home *** CD-ROM | disk | FTP | other *** search
/ PC World 2007 December / PCWorld_2007-12_cd.bin / v cisle / htttrack / httrack-3.41-3.exe / {app} / src / htsback.c < prev    next >
C/C++ Source or Header  |  2007-02-03  |  150KB  |  3,796 lines
  1. /* ------------------------------------------------------------ */
  2. /*
  3. HTTrack Website Copier, Offline Browser for Windows and Unix
  4. Copyright (C) Xavier Roche and other contributors
  5.  
  6. This program is free software; you can redistribute it and/or
  7. modify it under the terms of the GNU General Public License
  8. as published by the Free Software Foundation; either version 2
  9. of the License, or any later version.
  10.  
  11. This program is distributed in the hope that it will be useful,
  12. but WITHOUT ANY WARRANTY; without even the implied warranty of
  13. MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14. GNU General Public License for more details.
  15.  
  16. You should have received a copy of the GNU General Public License
  17. along with this program; if not, write to the Free Software
  18. Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
  19.  
  20.  
  21. Important notes:
  22.  
  23. - We hereby ask people using this source NOT to use it in purpose of grabbing
  24. emails addresses, or collecting any other private information on persons.
  25. This would disgrace our work, and spoil the many hours we spent on it.
  26.  
  27.  
  28. Please visit our Website: http://www.httrack.com
  29. */
  30.  
  31.  
  32. /* ------------------------------------------------------------ */
  33. /* File: httrack.c subroutines:                                 */
  34. /*       backing system (multiple socket download)              */
  35. /* Author: Xavier Roche                                         */
  36. /* ------------------------------------------------------------ */
  37.  
  38. /* Internal engine bytecode */
  39. #define HTS_INTERNAL_BYTECODE
  40.  
  41. /* specific definitions */
  42. #include "htsnet.h"
  43. #include "htscore.h"
  44. #include "htsthread.h"
  45. #include <time.h>
  46. /* END specific definitions */
  47.  
  48. #include "htsback.h"
  49.  
  50. //#ifdef _WIN32
  51. #include "htsftp.h"
  52. #if HTS_USEZLIB
  53. #include "htszlib.h"
  54. #else
  55. #error HTS_USEZLIB not defined
  56. #endif
  57. //#endif
  58.  
  59. #ifdef _WIN32
  60. #ifndef __cplusplus
  61. // DOS
  62. #ifndef  _WIN32_WCE
  63. #include <process.h>    /* _beginthread, _endthread */
  64. #endif
  65. #endif
  66. #else
  67. #endif
  68.  
  69. #if HTS_USEMMS
  70. #include "htsmms.h"
  71. #endif
  72.  
  73. #undef test_flush
  74. #define test_flush if (opt->flush) { if (opt->log) { fflush(opt->log); } if (opt->log) { fflush(opt->log);  } }
  75.  
  76. #define VT_CLREOL       "\33[K"
  77.  
  78. /* Slot operations */
  79. static int slot_can_be_cached_on_disk(const lien_back* back);
  80. static int slot_can_be_cleaned(const lien_back* back);
  81. static int slot_can_be_finalized(httrackp* opt, const lien_back* back);
  82.  
  83.  
  84. struct_back* back_new(int back_max) {
  85.   int i;
  86.   struct_back* sback = calloct(1, sizeof(struct_back));
  87.   sback->count = back_max;
  88.   sback->lnk = (lien_back*) calloct((back_max + 1), sizeof(lien_back));
  89.   sback->ready = inthash_new(32767);
  90.     sback->ready_size_bytes = 0;
  91.   inthash_value_is_malloc(sback->ready, 1);
  92.   // init
  93.   for(i = 0 ; i < sback->count ; i++){
  94.     sback->lnk[i].r.location = sback->lnk[i].location_buffer;
  95.     sback->lnk[i].status = STATUS_FREE;
  96.     sback->lnk[i].r.soc = INVALID_SOCKET;
  97.   }
  98.   return sback;
  99. }
  100.  
  101. void back_free(struct_back** sback) {
  102.   if (sback != NULL && *sback != NULL) {
  103.     if ((*sback)->lnk != NULL) {
  104.       freet((*sback)->lnk);
  105.       (*sback)->lnk = NULL;
  106.     }
  107.     if ((*sback)->ready != NULL) {
  108.       inthash_delete(&(*sback)->ready);
  109.             (*sback)->ready_size_bytes = 0;
  110.     }
  111.     freet(*sback);
  112.     *sback = NULL;
  113.   }
  114. }
  115.  
  116. void back_delete_all(httrackp* opt, cache_back* cache, struct_back* sback) {
  117.   if (sback != NULL) {
  118.     int i;
  119.     // delete live slots
  120.     for(i = 0 ; i < sback->count ; i++) {
  121.       back_delete(opt, cache, sback, i);
  122.     }
  123.     // delete stored slots
  124.     if (sback->ready != NULL) {
  125.       struct_inthash_enum e = inthash_enum_new(sback->ready);
  126.       inthash_chain* item;
  127.       while((item = inthash_enum_next(&e))) {
  128. #ifndef HTS_NO_BACK_ON_DISK
  129.                 char *filename = (char*) item->value.ptr;
  130.                 if (filename != NULL) {
  131.                     (void) unlink(filename);
  132.                 }
  133. #else
  134.                 /* clear entry content (but not yet the entry) */
  135.                 lien_back *back = (lien_back*) item->value.ptr;
  136.                 back_clear_entry(back);
  137. #endif
  138.       }
  139.             /* delete hashtable & content */
  140.             inthash_delete(&sback->ready);
  141.             sback->ready_size_bytes = 0;
  142.     }
  143.   }
  144. }
  145.  
  146. // ---
  147. // routines de backing
  148.  
  149. static int back_index_ready(httrackp* opt, struct_back* sback, char* adr, char* fil, char* sav, int getIndex);
  150. static int back_index_fetch(httrackp* opt, struct_back* sback, char* adr, char* fil, char* sav, int getIndex);
  151.  
  152. // retourne l'index d'un lien dans un tableau de backing
  153. int back_index(httrackp* opt, struct_back* sback,char* adr,char* fil,char* sav) {
  154.   return back_index_fetch(opt,sback, adr, fil, sav, 1);
  155. }
  156.  
  157. static int back_index_fetch(httrackp* opt, struct_back* sback, char* adr, char* fil, char* sav, int getIndex) {
  158.   lien_back* const back = sback->lnk;
  159.   const int back_max = sback->count;
  160.   int index=-1;
  161.   int i;
  162.   for( i = 0 ; i < back_max ; i++ ) {
  163.     if (back[i].status >= 0        /* not free or alive */
  164.       && strfield2(back[i].url_adr,adr)
  165.       && strcmp(back[i].url_fil,fil)==0) 
  166.     {
  167.       if (index==-1)    /* first time we meet, store it */
  168.         index=i;
  169.       else if (sav != NULL && strcmp(back[i].url_sav, sav) == 0) {  /* oops, check sav too */
  170.         index=i;
  171.         return index;
  172.       }
  173.     }
  174.   }
  175.   // not found in fast repository - search in the storage hashtable
  176.   if (index == -1 && sav != NULL) {
  177.     index = back_index_ready(opt, sback, adr, fil, sav, getIndex);
  178.   }
  179.   return index;
  180. }
  181.  
  182. /* resurrect stored entry */
  183. static int back_index_ready(httrackp* opt, struct_back* sback, char* adr, char* fil, char* sav, int getIndex) {
  184.   lien_back* const back = sback->lnk;
  185.   void* ptr = NULL;
  186.   if (inthash_read_pvoid(sback->ready, sav, &ptr)) {
  187.         if (!getIndex) {        /* don't "pagefault" the entry */
  188.             if (ptr != NULL) {
  189.                 return sback->count;        /* (invalid but) positive result */
  190.             } else {
  191.                 return -1;            /* not found */
  192.       }
  193.     } else if (ptr != NULL) {
  194.       lien_back* itemback = NULL;
  195. #ifndef HTS_NO_BACK_ON_DISK
  196.       FILE *fp;
  197.       char* fileback = (char*) ptr;
  198.         char catbuff[CATBUFF_SIZE];
  199.       if (( fp = fopen(fconv(catbuff, fileback), "rb") ) != NULL ) {
  200.         if (back_unserialize(fp, &itemback) != 0) {
  201.           if (itemback != NULL) {
  202.             back_clear_entry(itemback);
  203.             freet(itemback);
  204.             itemback = NULL;
  205.           }
  206.           if (opt->log != NULL) {
  207.             int last_errno = errno;
  208.             HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: unserialize error for %s%s (%s): %s"LF,adr,fil,sav,strerror(last_errno));
  209.             test_flush;
  210.           }
  211.         }
  212.         fclose(fp);
  213.       } else {
  214.         if (opt->log != NULL) {
  215.           int last_errno = errno;
  216.           HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: unserialize error for %s%s (%s), file disappeared: %s"LF,adr,fil,sav,strerror(last_errno));
  217.           test_flush;
  218.         }
  219.       }
  220.       (void) unlink(fileback);
  221. #else
  222.       itemback = (lien_back*) ptr;
  223. #endif
  224.             if (itemback != NULL) {
  225.                 // move from hashtable to fast repository
  226.                 int q = back_search(opt, sback);
  227.                 if (q != -1) {
  228.                     deletehttp(&back[q].r);               // security check
  229.                     back_move(itemback, &back[q]);
  230.                     back_clear_entry(itemback);                /* delete entry content */
  231.                     freet(itemback);                                    /* delete item */
  232.                     itemback = NULL;
  233.                     inthash_remove(sback->ready, sav);  // delete item
  234.                     sback->ready_size_bytes -= back[q].r.size;  /* substract for stats */
  235.           back_set_locked(sback, q);  /* locked */
  236.                     return q;
  237.         } else {
  238.           if (opt->log != NULL) {
  239.             HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: unserialize error for %s%s (%s): no more space to wakeup frozen slots"LF,adr,fil,sav);
  240.             test_flush;
  241.           }
  242.         }
  243.             }
  244.         }
  245.   }
  246.   return -1;
  247. }
  248.  
  249. static int slot_can_be_cached_on_disk(const lien_back* back) {
  250.   return
  251.     (back->status == STATUS_READY && back->locked == 0 
  252.       && back->url_sav[0] != '\0'
  253.       && strcmp(back->url_sav, BACK_ADD_TEST) != 0
  254.       );
  255.   /* Note: not checking !IS_DELAYED_EXT(back->url_sav) or it will quickly cause the slots to be filled! */
  256. }
  257.  
  258. /* Put all backing entries that are ready in the storage hashtable to spare space and CPU */
  259. int back_cleanup_background(httrackp* opt,cache_back* cache,struct_back* sback) {
  260.   lien_back* const back = sback->lnk;
  261.   const int back_max = sback->count;
  262.   int nclean = 0;
  263.   int i;
  264.   for( i = 0 ;  i < back_max ; i++ ) {
  265.     // ready, not locked and suitable
  266.     if (slot_can_be_cached_on_disk(&back[i])) {
  267. #ifdef HTS_NO_BACK_ON_DISK
  268.       lien_back* itemback;
  269. #endif
  270.       /* Security check */
  271.       int checkIndex = back_index_ready(opt, sback, back[i].url_adr, back[i].url_fil, back[i].url_sav, 1);
  272.       if (checkIndex != -1) {
  273.         if (opt->log) {
  274.           HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"engine: unexpected duplicate file entry: %s%s -> %s (%d '%s') / %s%s -> %s (%d '%s')"LF,
  275.             back[checkIndex].url_adr, back[checkIndex].url_fil, back[checkIndex].url_sav, back[checkIndex].r.statuscode, back[checkIndex].r.msg,
  276.             back[i].url_adr, back[i].url_fil, back[i].url_sav, back[i].r.statuscode, back[i].r.msg
  277.             );
  278.           test_flush;
  279.         }
  280.         back_delete(NULL, NULL, sback, checkIndex);
  281. #ifdef _DEBUG
  282.         /* This should NOT happend! */
  283.         { int duplicateEntryInBacklog = 1; assertf(!duplicateEntryInBacklog); }
  284. #endif
  285.             }
  286. #ifndef HTS_NO_BACK_ON_DISK
  287.             /* temporarily serialize the entry on disk */
  288.             {
  289.                 int fsz = (int) strlen(back[i].url_sav);
  290.         char *filename = malloc(fsz + 8 + 1);
  291.         if (filename != NULL) {
  292.           FILE *fp;
  293.           if (opt->getmode != 0) {
  294.             sprintf(filename, "%s.tmp", back[i].url_sav);
  295.           } else {
  296.             sprintf(filename, "%stmpfile%d.tmp", StringBuff(opt->path_html), opt->state.tmpnameid++);
  297.           }
  298.           /* Security check */
  299.           if (fexist(filename)) {
  300.             if (opt->log != NULL) {
  301.               HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: temporary file %s already exists"LF, filename);
  302.               test_flush;
  303.             }
  304.           }
  305.           /* Create file and serialize slot */
  306.           if ((fp = filecreate(NULL, filename)) != NULL) 
  307.           {
  308.             if (back_serialize(fp, &back[i]) == 0)
  309.             {
  310.               inthash_add_pvoid(sback->ready, back[i].url_sav, filename);
  311.               filename = NULL;
  312.               sback->ready_size_bytes += back[i].r.size;  /* add for stats */
  313.               nclean++;
  314.               back_clear_entry(&back[i]);            /* entry is now recycled */
  315.             } else {
  316.               if (opt->log != NULL) {
  317.                 int last_errno = errno;
  318.                 HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: serialize error for %s%s to %s: write error: %s"LF,back[i].url_adr,back[i].url_fil,filename,strerror(last_errno));
  319.                 test_flush;
  320.               }
  321.             }
  322.             fclose(fp);
  323.           } else {
  324.             if (opt->log != NULL) {
  325.               int last_errno = errno;
  326.               HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: serialize error for %s%s to %s: open error: %s (%s, %s)"LF, back[i].url_adr, back[i].url_fil, filename, strerror(last_errno), dir_exists(filename) ? "directory exists" : "directory does NOT exist!", fexist(filename) ? "file already exists!" : "file does not exist");
  327.               test_flush;
  328.             }
  329.           }
  330.                     if (filename != NULL)
  331.                         free(filename);
  332.                 } else {
  333.           if (opt->log != NULL) {
  334.             int last_errno = errno;
  335.             HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: serialize error for %s%s to %s: memory full: %s"LF,back[i].url_adr,back[i].url_fil,filename,strerror(last_errno));
  336.             test_flush;
  337.           }
  338.                 }
  339.             }
  340. #else
  341.             itemback = calloct(1, sizeof(lien_back));
  342.       back_move(&back[i], itemback);
  343.       inthash_add_pvoid(sback->ready, itemback->url_sav, itemback);
  344.       nclean++;
  345. #endif
  346.     }
  347.   }
  348.   return nclean;
  349. }
  350.  
  351. // nombre d'entrΘes libres dans le backing
  352. int back_available(struct_back* sback) {
  353.   lien_back* const back = sback->lnk;
  354.   const int back_max = sback->count;
  355.   int i;
  356.   int nb=0;
  357.   for(i=0;i<back_max;i++)
  358.     if (back[i].status==STATUS_FREE)     /* libre */
  359.       nb++;
  360.   return nb;
  361. }
  362.  
  363. // retourne estimation de la taille des html et fichiers stockΘs en mΘmoire
  364. LLint back_incache(struct_back* sback) {
  365.   lien_back* const back = sback->lnk;
  366.   const int back_max = sback->count;
  367.   int i;
  368.   LLint sum=0;
  369.   for(i=0;i<back_max;i++)
  370.     if (back[i].status!=-1)
  371.       if (back[i].r.adr)       // ne comptabilier que les blocs en mΘmoire
  372.         sum+=max(back[i].r.size,back[i].r.totalsize);
  373.   // stored (ready) slots
  374. #ifdef HTS_NO_BACK_ON_DISK
  375.   if (sback->ready != NULL) {
  376.     struct_inthash_enum e = inthash_enum_new(sback->ready);
  377.     inthash_chain* item;
  378.     while((item = inthash_enum_next(&e))) {
  379.       lien_back* ritem = (lien_back*) item->value.ptr;
  380.       if (ritem->status!=-1)
  381.         if (ritem->r.adr)       // ne comptabilier que les blocs en mΘmoire
  382.           sum+=max(ritem->r.size,ritem->r.totalsize);
  383.     }
  384.   }
  385. #endif
  386.   return sum;
  387. }
  388.  
  389. // retourne estimation de la taille des html et fichiers stockΘs en mΘmoire
  390. int back_done_incache(struct_back* sback) {
  391.   lien_back* const back = sback->lnk;
  392.   const int back_max = sback->count;
  393.   int i;
  394.   int n = 0;
  395.   for(i = 0 ; i < back_max ; i++)
  396.     if (back[i].status == STATUS_READY)
  397.       n++;
  398.   // stored (ready) slots
  399.   if (sback->ready != NULL) {
  400. #ifndef HTS_NO_BACK_ON_DISK
  401.         n += inthash_nitems(sback->ready);
  402. #else
  403.         struct_inthash_enum e = inthash_enum_new(sback->ready);
  404.     inthash_chain* item;
  405.     while((item = inthash_enum_next(&e))) {
  406.       lien_back* ritem = (lien_back*) item->value.ptr;
  407.       if (ritem->status==STATUS_READY)
  408.         n++;
  409.     }
  410. #endif
  411.   }
  412.   return n;
  413. }
  414.  
  415.  
  416. // le lien a-t-il ΘtΘ mis en backing?
  417. HTS_INLINE int back_exist(struct_back* sback,httrackp* opt,char* adr,char* fil,char* sav) {
  418.   return (back_index_fetch(opt, sback, adr, fil, sav, /*don't fetch*/0) >= 0);
  419. }
  420.  
  421. // nombre de sockets en tΓche de fond
  422. int back_nsoc(struct_back* sback) {
  423.   lien_back* const back = sback->lnk;
  424.   const int back_max = sback->count;
  425.   int n=0;
  426.   int i;
  427.   for(i=0;i<back_max;i++)
  428.     if (back[i].status > 0)    // only receive
  429.       n++;
  430.  
  431.   return n;
  432. }
  433. int back_nsoc_overall(struct_back* sback) {
  434.   lien_back* const back = sback->lnk;
  435.   const int back_max = sback->count;
  436.   int n=0;
  437.   int i;
  438.   for(i=0;i<back_max;i++)
  439.     if (back[i].status > 0 || back[i].status == STATUS_ALIVE)
  440.       n++;
  441.  
  442.   return n;
  443. }
  444.  
  445. // objet (lien) tΘlΘchargΘ ou transfΘrΘ depuis le cache
  446. //
  447. // fermer les paramΦtres de transfert,
  448. // et notamment vΘrifier les fichiers compressΘs (dΘcompresser), callback etc.
  449. int back_finalize(httrackp* opt,cache_back* cache,struct_back* sback,int p) {
  450.     char catbuff[CATBUFF_SIZE];
  451.   lien_back* const back = sback->lnk;
  452.   const int back_max = sback->count;
  453.   assertf(p >= 0 && p < back_max);
  454.  
  455.     /* Store ? */
  456.     if (!back[p].finalized) {
  457.         back[p].finalized = 1;
  458.  
  459.         /* Don't store broken files */
  460.         if (back[p].r.totalsize > 0 && back[p].r.size != back[p].r.totalsize && ! opt->tolerant) {
  461.             return -1;
  462.         }
  463.  
  464.         if (
  465.             (back[p].status == STATUS_READY)      // ready
  466.             &&
  467.             (back[p].r.statuscode > 0)   // not internal error
  468.             ) 
  469.         {
  470.             if (!back[p].testmode) {        // not test mode
  471.                 char* state="unknown";
  472.  
  473.                 /* dΘcompression */
  474. #if HTS_USEZLIB
  475.                 if (gz_is_available && back[p].r.compressed) {
  476.                     if (back[p].r.size > 0) {
  477.                         //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
  478.                         // stats
  479.                         back[p].compressed_size=back[p].r.size;
  480.                         // en mΘmoire -> passage sur disque
  481.                         if (!back[p].r.is_write) {
  482.                             back[p].tmpfile_buffer[0]='\0';
  483.                             back[p].tmpfile=tmpnam(back[p].tmpfile_buffer);
  484.                             if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') {
  485.                                 back[p].r.out=fopen(back[p].tmpfile,"wb");
  486.                                 if (back[p].r.out) {
  487.                                     if ((back[p].r.adr) && (back[p].r.size>0)) {
  488.                                         if (fwrite(back[p].r.adr,1,(size_t)back[p].r.size,back[p].r.out) != back[p].r.size) {
  489.                                             back[p].r.statuscode=STATUSCODE_INVALID;
  490.                                             strcpybuff(back[p].r.msg,"Write error when decompressing");
  491.                                         }
  492.                                     } else {
  493.                                         back[p].tmpfile[0]='\0';
  494.                                         back[p].r.statuscode=STATUSCODE_INVALID;
  495.                                         strcpybuff(back[p].r.msg,"Empty compressed file");
  496.                                     }
  497.                                 } else {
  498.                                     back[p].tmpfile[0]='\0';
  499.                                     back[p].r.statuscode=STATUSCODE_INVALID;
  500.                                     strcpybuff(back[p].r.msg,"Open error when decompressing");
  501.                                 }
  502.                             }
  503.                         }
  504.                         // fermer fichier sortie
  505.                         if (back[p].r.out!=NULL) {
  506.                             fclose(back[p].r.out);
  507.                             back[p].r.out=NULL;
  508.                         }
  509.                         // dΘcompression
  510.                         if (back[p].tmpfile != NULL && back[p].tmpfile[0] != '\0') {
  511.                             if (back[p].url_sav[0]) {
  512.                                 LLint size;
  513.                                 file_notify(opt,back[p].url_adr, back[p].url_fil, back[p].url_sav, 1, 1, back[p].r.notmodified);
  514.                                 filecreateempty(&opt->state.strc, back[p].url_sav);      // filenote & co
  515.                                 if ((size = hts_zunpack(back[p].tmpfile,back[p].url_sav))>=0) {
  516.                                     back[p].r.size=back[p].r.totalsize=size;
  517.                                     // fichier -> mΘmoire
  518.                                     if (!back[p].r.is_write) {
  519.                                         deleteaddr(&back[p].r);
  520.                                         back[p].r.adr=readfile(back[p].url_sav);
  521.                                         if (!back[p].r.adr) {
  522.                                             back[p].r.statuscode=STATUSCODE_INVALID;
  523.                                             strcpybuff(back[p].r.msg,"Read error when decompressing");
  524.                                         }
  525.                                         unlink(back[p].url_sav);
  526.                                     }
  527.                                 }
  528.                             }
  529.                             /* encore that no remaining temporary file exists */
  530.                             unlink(back[p].tmpfile);
  531.                             back[p].tmpfile = NULL;
  532.                         }
  533.                         // stats
  534.                         HTS_STAT.total_packed+=back[p].compressed_size;
  535.                         HTS_STAT.total_unpacked+=back[p].r.size;
  536.                         HTS_STAT.total_packedfiles++;
  537.                         // unflag
  538.                     }
  539.                 }
  540.                 back[p].r.compressed=0;
  541. #endif
  542.  
  543.                 /* Write mode to disk */
  544.                 if (back[p].r.is_write && back[p].r.adr != NULL) {
  545.                     freet(back[p].r.adr);
  546.                     back[p].r.adr = NULL;
  547.                 }
  548.  
  549.                 /* ************************************************************************
  550.                     REAL MEDIA HACK
  551.                     Check if we have to load locally the file
  552.                 ************************************************************************ */
  553.                 if (back[p].r.statuscode == HTTP_OK) {    // OK (ou 304 en backing)
  554.                     if (back[p].r.is_write) {    // Written file
  555.                         if (may_be_hypertext_mime(opt,back[p].r.contenttype, back[p].url_fil)) {   // to parse!
  556.                             off_t sz;
  557.                             sz=fsize(back[p].url_sav);
  558.                             if (sz>0) {   // ok, exists!
  559.                                 if (sz < 8192) {   // ok, small file --> to parse!
  560.                                     FILE* fp=fopen(back[p].url_sav,"rb");
  561.                                     if (fp) {
  562.                                         back[p].r.adr=malloct((int)sz + 2);
  563.                                         if (back[p].r.adr) {
  564.                                             if (fread(back[p].r.adr,1,sz,fp) == sz) {
  565.                                                 back[p].r.size=sz;
  566.                                                 back[p].r.adr[sz] = '\0';
  567.                                                 back[p].r.is_write = 0;                /* not anymore a direct-to-disk file */
  568.                                             } else {
  569.                                                 freet(back[p].r.adr);
  570.                                                 back[p].r.size=0;
  571.                                                 back[p].r.adr = NULL;
  572.                                                 back[p].r.statuscode=STATUSCODE_INVALID;
  573.                                                 strcpybuff(back[p].r.msg, ".RAM read error");
  574.                                             }
  575.                                             fclose(fp);
  576.                                             fp=NULL;
  577.                                             // remove (temporary) file!
  578.                                             unlink(fconv(catbuff,back[p].url_sav));
  579.                                         }
  580.                                         if (fp)
  581.                                             fclose(fp);
  582.                                     }
  583.                                 }
  584.                             }
  585.                         }
  586.                     }
  587.                 }
  588.                 /* EN OF REAL MEDIA HACK */
  589.  
  590.  
  591.                 /* Stats */
  592.                 if (cache->txt) {
  593.                     char flags[32];
  594.                     char s[256];
  595.                     time_t tt;
  596.                     struct tm* A;
  597.                     tt=time(NULL);
  598.                     A=localtime(&tt);
  599.                     if (A == NULL) {
  600.                         int localtime_returned_null=0;
  601.                         assert(localtime_returned_null);
  602.                     }
  603.                     strftime(s,250,"%H:%M:%S",A);
  604.  
  605.                     flags[0]='\0';
  606.                     /* input flags */
  607.                     if (back[p].is_update)
  608.                         strcatbuff(flags, "U");   // update request
  609.                     else
  610.                         strcatbuff(flags, "-");
  611.                     if (back[p].range_req_size)
  612.                         strcatbuff(flags, "R");   // range request
  613.                     else
  614.                         strcatbuff(flags, "-");
  615.                     /* state flags */
  616.                     if (back[p].r.is_file)  // direct to disk
  617.                         strcatbuff(flags, "F");
  618.                     else
  619.                         strcatbuff(flags, "-");
  620.                     /* output flags */
  621.                     if (!back[p].r.notmodified)
  622.                         strcatbuff(flags, "M");   // modified
  623.                     else
  624.                         strcatbuff(flags, "-");
  625.                     if (back[p].r.is_chunk)  // chunked
  626.                         strcatbuff(flags, "C");
  627.                     else
  628.                         strcatbuff(flags, "-");
  629.                     if (back[p].r.compressed)
  630.                         strcatbuff(flags, "Z");   // gzip
  631.                     else
  632.                         strcatbuff(flags, "-");
  633.                     /* Err I had to split these.. */
  634.                     fprintf(cache->txt,"%s\t", s);
  635.                     fprintf(cache->txt,LLintP"/", (LLint)back[p].r.size);
  636.                     fprintf(cache->txt,LLintP,(LLint)back[p].r.totalsize);
  637.                     fprintf(cache->txt,"\t%s\t",flags);
  638.                 }
  639.                 if (back[p].r.statuscode == HTTP_OK) {
  640.                     if (back[p].r.size>=0) {
  641.                         if (strcmp(back[p].url_fil,"/robots.txt") !=0 ) {
  642.                             HTS_STAT.stat_bytes+=back[p].r.size;
  643.                             HTS_STAT.stat_files++;
  644.                         }
  645.                         if ( (!back[p].r.notmodified) && (opt->is_update) ) { 
  646.                             HTS_STAT.stat_updated_files++;       // page modifiΘe
  647.                             if (opt->log!=NULL) {
  648.                                 HTS_LOG(opt,LOG_INFO);
  649.                                 if (back[p].is_update) {
  650.                                     fprintf(opt->log,"engine: transfer-status: link updated: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  651.                                 } else {
  652.                                     fprintf(opt->log,"engine: transfer-status: link added: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  653.                                 }
  654.                                 test_flush;
  655.                             }
  656.                             if (cache->txt) {
  657.                                 if (back[p].is_update) {
  658.                                     state="updated";
  659.                                 } else {
  660.                                     state="added";
  661.                                 }
  662.                             }
  663.                         } else {
  664.                             if ( (opt->debug>0) && (opt->log!=NULL) ) {
  665.                                 HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: transfer-status: link recorded: %s%s -> %s"LF,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  666.                                 test_flush;
  667.                             }
  668.                             if (cache->txt) {
  669.                                 if (opt->is_update)
  670.                                     state="untouched";
  671.                                 else
  672.                                     state="added";
  673.                             }
  674.                         }
  675.                     } else {
  676.                         if ( (opt->debug>0) && (opt->log!=NULL) ) {
  677.                             HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: transfer-status: empty file? (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil);
  678.                             test_flush;
  679.                         }
  680.                         if (cache->txt) {
  681.                             state="empty";
  682.                         }
  683.                     }
  684.                 } else {
  685.                     if ( (opt->debug>0) && (opt->log!=NULL) ) {
  686.                         HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: transfer-status: link error (%d, '%s'): %s%s"LF,back[p].r.statuscode,back[p].r.msg,back[p].url_adr,back[p].url_fil);
  687.                     }
  688.                     if (cache->txt) {
  689.                         state="error";
  690.                     }
  691.                 }
  692.                 if (cache->txt) {
  693.                     fprintf(cache->txt,
  694.                         "%d\t"
  695.                         "%s ('%s')\t"
  696.                         "%s\t"
  697.                         "%s%s\t"
  698.                         "%s%s%s\t%s\t"
  699.                         "(from %s%s%s)"
  700.                         LF,
  701.                         back[p].r.statuscode,
  702.                         state, escape_check_url_addr(OPT_GET_BUFF(opt),back[p].r.msg),
  703.                         escape_check_url_addr(OPT_GET_BUFF(opt),back[p].r.contenttype),
  704.                         ((back[p].r.etag[0])?"etag:":((back[p].r.lastmodified[0])?"date:":"")), escape_check_url_addr(OPT_GET_BUFF(opt),(back[p].r.etag[0])?back[p].r.etag:(back[p].r.lastmodified)),
  705.                         (link_has_authority(back[p].url_adr) ? "" : "http://"),escape_check_url_addr(OPT_GET_BUFF(opt),back[p].url_adr),escape_check_url_addr(OPT_GET_BUFF(opt),back[p].url_fil),escape_check_url_addr(OPT_GET_BUFF(opt),back[p].url_sav),
  706.                         (link_has_authority(back[p].referer_adr) || !back[p].referer_adr[0]) ? "" : "http://",escape_check_url_addr(OPT_GET_BUFF(opt),back[p].referer_adr),escape_check_url_addr(OPT_GET_BUFF(opt),back[p].referer_fil)
  707.                         );
  708.                     if (opt->flush)
  709.                         fflush(cache->txt);
  710.                 }
  711.  
  712.                 /* Cache */
  713.                 if (!IS_DELAYED_EXT(back[p].url_sav)) {
  714.                     cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,back[p].url_sav);
  715.                 } else {
  716.                     if (!HTTP_IS_OK(back[p].r.statuscode)) {
  717.                         if ( (opt->debug>0) && (opt->log!=NULL) ) {
  718.                             HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"redirect to %s%s"LF,back[p].url_adr,back[p].url_fil);
  719.                         }
  720.                         /* Store only header reference */
  721.                         cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL);
  722.                     } else {
  723.             /* Partial file, but marked as "ok" ? */
  724.                         if (opt->log!=NULL) {
  725.                             HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"file not stored in cache due to bogus state (incomplete type): %s%s"LF,back[p].url_adr,back[p].url_fil);
  726.                         }
  727.                     }
  728.                 }
  729.  
  730.                 // status finished callback
  731.             RUN_CALLBACK1(opt, xfrstatus, &back[p]);
  732.  
  733.                 return 0;
  734.             } else {        // testmode
  735.                 if (back[p].r.statuscode / 100 >= 3) {        /* Store 3XX, 4XX, 5XX test response codes, but NOT 2XX */
  736.                     /* Cache */
  737.                     cache_mayadd(opt,cache,&back[p].r,back[p].url_adr,back[p].url_fil,NULL);
  738.                 }
  739.             }
  740.         }
  741.     }
  742.     return -1;
  743. }
  744.  
  745. /* try to keep the connection alive */
  746. int back_letlive(httrackp* opt, cache_back* cache, struct_back* sback, int p) {
  747.     lien_back* const back = sback->lnk;
  748.     const int back_max = sback->count;
  749.     int checkerror;
  750.     htsblk* src = &back[p].r;
  751.     assertf(p >= 0 && p < back_max);
  752.     if (src && !src->is_file 
  753.         && src->soc != INVALID_SOCKET
  754.         && src->statuscode >= 0           /* no timeout errors & co */
  755.         && src->keep_alive_trailers == 0  /* not yet supported (chunk trailers) */
  756.         && ! ( checkerror = check_sockerror(src->soc) )
  757.         /*&& !check_sockdata(src->soc)*/     /* no unexpected data */
  758.         ) {
  759.             htsblk tmp;
  760.             memset(&tmp, 0, sizeof(tmp));
  761.             /* clear everything but connection: switch, close, and reswitch */
  762.             back_connxfr(src, &tmp);
  763.             back_delete(opt, cache, sback, p);
  764.             //deletehttp(src);
  765.             back_connxfr(&tmp, src);
  766.             src->req.flush_garbage=1;     /* ignore CRLF garbage */
  767.             return 1;
  768.         }
  769.         return 0;
  770. }
  771.  
  772. void back_connxfr(htsblk* src, htsblk* dst) {
  773.   dst->soc = src->soc;
  774.   src->soc = INVALID_SOCKET;
  775. #if HTS_USEOPENSSL
  776.   dst->ssl = src->ssl;
  777.   src->ssl = 0;
  778.   dst->ssl_con = src->ssl_con;
  779.   src->ssl_con = NULL;
  780. #endif
  781.   dst->keep_alive = src->keep_alive;
  782.   src->keep_alive = 0;
  783.   dst->keep_alive_max = src->keep_alive_max;
  784.   src->keep_alive_max = 0;
  785.   dst->keep_alive_t = src->keep_alive_t;
  786.   src->keep_alive_t = 0;
  787.   dst->debugid = src->debugid;
  788.   src->debugid = 0;
  789. }
  790.  
  791. void back_move(lien_back* src, lien_back* dst) {
  792.   memcpy(dst, src, sizeof(lien_back));
  793.   memset(src, 0, sizeof(lien_back));  
  794.   src->r.soc=INVALID_SOCKET;
  795.   src->status=STATUS_FREE;
  796.   src->r.location = src->location_buffer;
  797.   dst->r.location = dst->location_buffer;
  798. }
  799.  
  800. void back_copy_static(const lien_back* src, lien_back* dst) {
  801.   memcpy(dst, src, sizeof(lien_back));
  802.   dst->r.soc=INVALID_SOCKET;
  803.     dst->r.adr = NULL;
  804.     dst->r.headers = NULL;
  805.     dst->r.out = NULL;
  806.   dst->r.location = dst->location_buffer;
  807.     dst->r.fp = NULL;
  808. #if HTS_USEOPENSSL
  809.     dst->r.ssl_con = NULL;
  810. #endif
  811. }
  812.  
  813. static int back_data_serialize(FILE *fp, const void *data, size_t size) {
  814.     if ( fwrite(&size, 1, sizeof(size), fp) == sizeof(size)
  815.         && ( size == 0 || fwrite(data, 1, size, fp) == size )
  816.         )
  817.         return 0;
  818.     return 1;    /* error */
  819. }
  820.  
  821. static int back_string_serialize(FILE *fp, const char *str) {
  822.     size_t size = ( str != NULL ) ? ( strlen(str) + 1 ) : 0;
  823.     return back_data_serialize(fp, str, size);
  824. }
  825.  
  826. static int back_data_unserialize(FILE *fp, void **str, size_t *size) {
  827.     *str = NULL;
  828.     if (fread(size, 1, sizeof(*size), fp) == sizeof(*size)) {
  829.         if (*size == 0)            /* serialized NULL ptr */
  830.             return 0;
  831.         *str = malloct(*size + 1);
  832.         if (*str == NULL)
  833.             return 1;        /* error */
  834.         ((char*) *str)[*size] = 0;        /* guard byte */
  835.         if (fread(*str, 1, *size, fp) == *size)
  836.             return 0;
  837.     }
  838.     return 1;        /* error */
  839. }
  840.  
  841. static int back_string_unserialize(FILE *fp, char **str) {
  842.     size_t dummy;
  843.     return back_data_unserialize(fp, (void**) str, &dummy);
  844. }
  845.  
  846. int back_serialize(FILE *fp, const lien_back* src) {
  847.     if (back_data_serialize(fp, src, sizeof(lien_back)) == 0
  848.         && back_data_serialize(fp, src->r.adr, src->r.adr ? (size_t)src->r.size : 0) == 0
  849.         && back_string_serialize(fp, src->r.headers) == 0
  850.         && fflush(fp) == 0)
  851.         return 0;
  852.     return 1;
  853. }
  854.  
  855. int back_unserialize(FILE *fp, lien_back** dst) {
  856.     size_t size;
  857.     *dst = NULL;
  858.   errno = 0;
  859.     if (back_data_unserialize(fp, (void**) dst, &size) == 0 && size == sizeof(lien_back)) {
  860.         (*dst)->tmpfile = NULL;
  861.         (*dst)->chunk_adr = NULL;
  862.         (*dst)->r.adr = NULL;
  863.         (*dst)->r.out = NULL;
  864.         (*dst)->r.location = (*dst)->location_buffer;
  865.         (*dst)->r.fp = NULL;
  866. #if HTS_USEOPENSSL
  867.         (*dst)->r.ssl_con = NULL;
  868. #endif
  869.         if (back_data_unserialize(fp, (void**) &(*dst)->r.adr, &size) == 0) 
  870.         {
  871.             (*dst)->r.size = size;
  872.             (*dst)->r.headers = NULL;
  873.             if (back_string_unserialize(fp, &(*dst)->r.headers) == 0)
  874.                 return 0;        /* ok */
  875.             if ((*dst)->r.headers != NULL)
  876.                 freet((*dst)->r.headers);
  877.         }
  878.         if ((*dst)->r.adr != NULL)
  879.             freet((*dst)->r.adr);
  880.     }
  881.     if (dst != NULL)
  882.         freet(dst);
  883.     *dst = NULL;
  884.     return 1;        /* error */
  885. }
  886.  
  887. // clear, or leave for keep-alive
  888. int back_maydelete(httrackp* opt,cache_back* cache,struct_back* sback, int p) {
  889.   lien_back* const back = sback->lnk;
  890.   const int back_max = sback->count;
  891.   assertf(p >= 0 && p < back_max);
  892.   if (p >= 0 && p < back_max) {    // on sait jamais..
  893.     if (
  894.       /* Keep-alive authorized by user */
  895.       !opt->nokeepalive
  896.       /* Socket currently is keep-alive! */
  897.       && back[p].r.keep_alive 
  898.       /* Remaining authorized requests */
  899.       && back[p].r.keep_alive_max > 1
  900.       /* Known keep-alive start (security) */
  901.       && back[p].ka_time_start 
  902.       /* We're on time */
  903.       && time_local() < back[p].ka_time_start + back[p].r.keep_alive_t
  904.       /* Connection delay must not exceed keep-alive timeout */
  905.       && ( opt->maxconn <= 0 || ( back[p].r.keep_alive_t > ( 1.0 / opt->maxconn ) ) )
  906.       ) {
  907.       lien_back tmp;
  908.       strcpybuff(tmp.url_adr, back[p].url_adr);
  909.       if (back_letlive(opt, cache, sback, p)) {
  910.         strcpybuff(back[p].url_adr, tmp.url_adr);
  911.         back[p].status = STATUS_ALIVE;  // alive & waiting
  912.         if ((opt->debug>1) && (opt->log!=NULL)) {
  913.           HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"(Keep-Alive): successfully saved #%d (%s)"LF, 
  914.             back[p].r.debugid,
  915.             back[p].url_adr); test_flush;
  916.         }
  917.         return 1;
  918.       }
  919.     }
  920.     back_delete(opt,cache,sback, p);
  921.   }
  922.   return 0;
  923. }
  924.  
  925. // clear, or leave for keep-alive
  926. void back_maydeletehttp(httrackp* opt, cache_back* cache, struct_back* sback, int p) {
  927.   lien_back* const back = sback->lnk;
  928.   const int back_max = sback->count;
  929.   TStamp lt = 0;
  930.   assertf(p >= 0 && p < back_max);
  931.   if (back[p].r.soc!=INVALID_SOCKET) {
  932.     int q;
  933.     if (
  934.       back[p].r.soc != INVALID_SOCKET        /* security check */
  935.       && back[p].r.statuscode >= 0           /* no timeout errors & co */
  936.       && back[p].r.keep_alive_trailers == 0  /* not yet supported (chunk trailers) */
  937.       /* Socket not in I/O error status */
  938.       && !back[p].r.is_file
  939.       && !check_sockerror(back[p].r.soc)
  940.       /* Keep-alive authorized by user */
  941.       && !opt->nokeepalive
  942.       /* Socket currently is keep-alive! */
  943.       && back[p].r.keep_alive 
  944.       /* Remaining authorized requests */
  945.       && back[p].r.keep_alive_max > 1
  946.       /* Known keep-alive start (security) */
  947.       && back[p].ka_time_start 
  948.       /* We're on time */
  949.       && ( lt = time_local() ) < back[p].ka_time_start + back[p].r.keep_alive_t
  950.       /* Connection delay must not exceed keep-alive timeout */
  951.       && ( opt->maxconn <= 0 || ( back[p].r.keep_alive_t > ( 1.0 / opt->maxconn ) ) )
  952.       /* Available slot in backing */
  953.       && ( q = back_search(opt, sback) ) >= 0
  954.       ) 
  955.     {
  956.       lien_back tmp;
  957.       strcpybuff(tmp.url_adr, back[p].url_adr);
  958.       deletehttp(&back[q].r);               // security check
  959.       back_connxfr(&back[p].r, &back[q].r); // transfer live connection settings from p to q
  960.       back[q].ka_time_start = back[p].ka_time_start;  // refresh
  961.       back[p].r.soc = INVALID_SOCKET;
  962.       strcpybuff(back[q].url_adr, tmp.url_adr); // address
  963.       back[q].status = STATUS_ALIVE;  // alive & waiting
  964.       if ((opt->debug>1) && (opt->log!=NULL)) {
  965.         HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"(Keep-Alive): successfully preserved #%d (%s)"LF, 
  966.           back[q].r.debugid,
  967.           back[q].url_adr); test_flush;
  968.       }
  969.     } else {
  970.       deletehttp(&back[p].r);
  971.       back[p].r.soc = INVALID_SOCKET;
  972.     }
  973.   }
  974. }
  975.  
  976.  
  977. /* attempt to attach a live connection to this slot */
  978. int back_trylive(httrackp* opt,cache_back* cache,struct_back* sback, int p) {
  979.   lien_back* const back = sback->lnk;
  980.   const int back_max = sback->count;
  981.   assertf(p >= 0 && p < back_max);
  982.   if (p>=0 && back[p].status != STATUS_ALIVE) {     // we never know..
  983.     int i = back_searchlive(opt,sback, back[p].url_adr);   // search slot
  984.     if (i >= 0 && i != p) {
  985.       deletehttp(&back[p].r);               // security check
  986.       back_connxfr(&back[i].r, &back[p].r); // transfer live connection settings from i to p
  987.       back_delete(opt,cache,sback, i);      // delete old slot
  988.       back[p].status=STATUS_CONNECTING;     // ready to connect
  989.       return 1;                             // success: will reuse live connection
  990.     }
  991.   }
  992.   return 0;
  993. }
  994.  
  995. /* search for a live position, or, if not possible, try to return a new one */
  996. int back_searchlive(httrackp* opt, struct_back* sback, char* search_addr) {
  997.   lien_back* const back = sback->lnk;
  998.   const int back_max = sback->count;
  999.   int i;
  1000.  
  1001.   /* search for a live socket */
  1002.   for(i = 0 ; i < back_max ; i++ ) {
  1003.     if (back[i].status == STATUS_ALIVE) {
  1004.       if (strfield2(back[i].url_adr, search_addr)) {   /* same location (xxc: check also virtual hosts?) */
  1005.         if (time_local() < back[i].ka_time_start + back[i].r.keep_alive_t) {
  1006.           return i;
  1007.         }
  1008.       }
  1009.     }
  1010.   }
  1011.   return -1;
  1012. }
  1013.   
  1014. int back_search_quick(struct_back* sback) {
  1015.   lien_back* const back = sback->lnk;
  1016.   const int back_max = sback->count;
  1017.   int i;
  1018.  
  1019.   /* try to find an empty place */
  1020.   for(i = 0 ; i < back_max ; i++ ) {
  1021.     if (back[i].status == STATUS_FREE) {
  1022.       return i;
  1023.     }
  1024.   }
  1025.  
  1026.   /* oops, can't find a place */
  1027.   return -1;
  1028. }
  1029.  
  1030. int back_search(httrackp* opt,struct_back* sback) {
  1031.   lien_back* const back = sback->lnk;
  1032.   const int back_max = sback->count;
  1033.   int i;
  1034.  
  1035.   /* try to find an empty place */
  1036.   if ( ( i = back_search_quick(sback) ) != -1)
  1037.     return i;
  1038.  
  1039.   /* couldn't find an empty place, try to requisition a keep-alive place */
  1040.   for(i = 0 ; i < back_max ; i++ ) {
  1041.     if (back[i].status == STATUS_ALIVE) {
  1042.       lien_back* const back = sback->lnk;
  1043.       /* close this place */
  1044.       back_clear_entry(&back[i]);   /* Already finalized (this is the night of the living dead) */
  1045.       /*back_delete(opt,cache,sback, i);*/
  1046.       return i;
  1047.     }
  1048.   }
  1049.  
  1050.   /* oops, can't find a place */
  1051.   return -1;
  1052. }
  1053.  
  1054. void back_set_finished(struct_back* sback, int p) {
  1055.   lien_back* const back = sback->lnk;
  1056.   const int back_max = sback->count;
  1057.   assertf(p >= 0 && p < back_max);
  1058.   if (p >= 0 && p < sback->count) {    // we never know..
  1059.     /* status: finished (waiting to be validated) */
  1060.     back[p].status=STATUS_READY;     /* finished */
  1061.     /* close open r/w streams, if any */
  1062.     if (back[p].r.fp!=NULL) {
  1063.       fclose(back[p].r.fp);
  1064.       back[p].r.fp=NULL;
  1065.     }
  1066.     if (back[p].r.out!=NULL) {  // fermer fichier sortie
  1067.       fclose(back[p].r.out);
  1068.       back[p].r.out=NULL;
  1069.     }
  1070.   }
  1071. }
  1072.  
  1073. void back_set_locked(struct_back* sback, int p) {
  1074.   lien_back* const back = sback->lnk;
  1075.   const int back_max = sback->count;
  1076.   assertf(p >= 0 && p < back_max);
  1077.   if (p >= 0 && p < sback->count) {
  1078.     /* status: locked (in process, do not swap on disk) */
  1079.     back[p].locked = 1;     /* locked */
  1080.   }
  1081. }
  1082.  
  1083. void back_set_unlocked(struct_back* sback, int p) {
  1084.   lien_back* const back = sback->lnk;
  1085.   const int back_max = sback->count;
  1086.   assertf(p >= 0 && p < back_max);
  1087.   if (p >= 0 && p < sback->count) {
  1088.     /* status: unlocked (can be swapped on disk) */
  1089.     back[p].locked = 0;     /* unlocked */
  1090.   }
  1091. }
  1092.  
  1093. int back_flush_output(httrackp* opt, cache_back* cache, struct_back* sback, int p) {
  1094.   lien_back* const back = sback->lnk;
  1095.   const int back_max = sback->count;
  1096.   assertf(p >= 0 && p < back_max);
  1097.   if (p >= 0 && p < sback->count) {    // on sait jamais..
  1098.         /* close input file */
  1099.         if (back[p].r.fp!=NULL) {
  1100.       fclose(back[p].r.fp);
  1101.       back[p].r.fp=NULL;
  1102.     }
  1103.     /* fichier de sortie */
  1104.     if (back[p].r.out!=NULL) {  // fermer fichier sortie
  1105.       fclose(back[p].r.out);
  1106.       back[p].r.out=NULL;
  1107.     }
  1108.         /* set file time */
  1109.     if (back[p].r.is_write) {     // ecriture directe
  1110.       /* Θcrire date "remote" */
  1111.       if (strnotempty(back[p].url_sav)
  1112.                 && strnotempty(back[p].r.lastmodified)
  1113.                 && fexist(back[p].url_sav))          // normalement existe si on a un fichier de sortie
  1114.             {
  1115.         set_filetime_rfc822(back[p].url_sav,back[p].r.lastmodified);
  1116.             }
  1117.       /* executer commande utilisateur aprΦs chargement du fichier */
  1118.       //xx usercommand(opt,0,NULL,back[p].url_sav, back[p].url_adr, back[p].url_fil);
  1119.       back[p].r.is_write=0;
  1120.     }
  1121.         return 1;
  1122.     }
  1123.     return 0;
  1124. }
  1125.  
  1126. // effacer entrΘe
  1127. int back_delete(httrackp* opt, cache_back* cache, struct_back* sback, int p) {
  1128.   lien_back* const back = sback->lnk;
  1129.   const int back_max = sback->count;
  1130.   assertf(p >= 0 && p < back_max);
  1131.   if (p >= 0 && p < sback->count) {    // on sait jamais..
  1132.     // VΘrificateur d'intΘgritΘ
  1133. #if DEBUG_CHECKINT
  1134.     _CHECKINT(&back[p],"Appel back_delete")
  1135. #endif
  1136. #if HTS_DEBUG_CLOSESOCK
  1137.       DEBUG_W("back_delete: #%d\n" _ (int) p);
  1138. #endif
  1139.  
  1140.     // Finalize
  1141.     if (!back[p].finalized) {
  1142.       if (
  1143.         (back[p].status == STATUS_READY)      // ready
  1144.         &&
  1145.         (!back[p].testmode)        // not test mode
  1146.         &&
  1147.         (back[p].r.statuscode>0)   // not internal error
  1148.         ) {
  1149.           if (opt != NULL && opt->debug>1 && opt->log!=NULL) {
  1150.             HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File '%s%s' -> %s not yet saved in cache - saving now"LF, back[p].url_adr, back[p].url_fil, back[p].url_sav); test_flush;
  1151.           }
  1152.         }
  1153.         if (cache != NULL) {
  1154.           back_finalize(opt, cache, sback, p);
  1155.         }
  1156.     }
  1157.     back[p].finalized = 0;
  1158.  
  1159.         // flush output buffers
  1160.         (void) back_flush_output(opt, cache, sback, p);
  1161.  
  1162.         return back_clear_entry(&back[p]);
  1163.   }
  1164.   return 0;
  1165. }
  1166.  
  1167. /* ensure that the entry is not locked */
  1168. void back_index_unlock(struct_back* sback, int p) {
  1169.   lien_back* const back = sback->lnk;
  1170.   if (back[p].locked) {
  1171.     back[p].locked = 0;   /* not locked anymore */
  1172.   }
  1173. }
  1174.  
  1175. /* the entry is available again */
  1176. static void back_set_free(lien_back* back) {
  1177.   back->locked = 0;
  1178.   back->status = STATUS_FREE;
  1179. }
  1180.  
  1181. /* delete entry content (clear the entry), but don't unallocate the entry itself */
  1182. int back_clear_entry(lien_back* back) {
  1183.     if (back != NULL) {
  1184.     // LibΘrer tous les sockets, handles, buffers..
  1185.     if (back->r.soc!=INVALID_SOCKET) {
  1186. #if HTS_DEBUG_CLOSESOCK
  1187.       DEBUG_W("back_delete: deletehttp\n");
  1188. #endif
  1189.       deletehttp(&back->r);
  1190.       back->r.soc=INVALID_SOCKET;
  1191.     }
  1192.     
  1193.     if (back->r.adr!=NULL) {  // reste un bloc α dΘsallouer
  1194.       freet(back->r.adr);
  1195.       back->r.adr=NULL;
  1196.     }
  1197.     if (back->chunk_adr!=NULL) {  // reste un bloc α dΘsallouer
  1198.       freet(back->chunk_adr);
  1199.       back->chunk_adr=NULL;
  1200.       back->chunk_size=0;
  1201.       back->chunk_blocksize=0;
  1202.       back->is_chunk=0;
  1203.     }
  1204.  
  1205.         // only for security
  1206.         if (back->tmpfile && back->tmpfile[0] != '\0') {
  1207.             (void) unlink(back->tmpfile);
  1208.             back->tmpfile = NULL;
  1209.         }
  1210.  
  1211.     // headers
  1212.     if (back->r.headers != NULL) {
  1213.       freet(back->r.headers);
  1214.       back->r.headers = NULL;
  1215.     }
  1216.  
  1217.     // Tout nettoyer
  1218.     memset(back, 0, sizeof(lien_back));  
  1219.     back->r.soc = INVALID_SOCKET;
  1220.         back->r.location = back->location_buffer;
  1221.     
  1222.     // Le plus important: libΘrer le champ
  1223.     back_set_free(back);
  1224.  
  1225.       return 1;
  1226.   }
  1227.   return 0;
  1228. }
  1229.  
  1230. /* Space left on backing stack */
  1231. int back_stack_available(struct_back* sback) {
  1232.   lien_back* const back = sback->lnk;
  1233.   const int back_max = sback->count;
  1234.   int p=0,n=0;
  1235.   for( ; p < back_max ; p++ )
  1236.     if ( back[p].status == STATUS_FREE )
  1237.       n++;
  1238.   return n;
  1239. }
  1240.  
  1241. // ajouter un lien en backing
  1242. int back_add_if_not_exists(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test) {
  1243.   back_clean(opt, cache, sback);  /* first cleanup the backlog to ensure that we have some entry left */
  1244.   if (!back_exist(sback,opt,adr,fil,save)) {
  1245.     return back_add(sback, opt, cache, adr, fil, save, referer_adr, referer_fil, test);
  1246.     }
  1247.   return 0;
  1248. }
  1249.  
  1250. int back_add(struct_back* sback,httrackp* opt,cache_back* cache,char* adr,char* fil,char* save,char* referer_adr,char* referer_fil,int test) {
  1251.   lien_back* const back = sback->lnk;
  1252.   const int back_max = sback->count;
  1253.   int p=0;
  1254.     char catbuff[CATBUFF_SIZE];
  1255.     char catbuff2[CATBUFF_SIZE];
  1256.  
  1257. #if (defined(_DEBUG) || defined(DEBUG))
  1258.   if (!test && back_exist(sback,opt,adr,fil,save)) {
  1259.     int already_there = 0;
  1260.     if (opt->log!=NULL) {
  1261.       HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"error: back_add(%s,%s,%s) duplicate"LF, adr, fil, save);
  1262.     }
  1263.   }
  1264. #endif
  1265.  
  1266.   // vΘrifier cohΘrence de adr et fil (non vide!)
  1267.   if (strnotempty(adr)==0) {
  1268.     if (opt->log!=NULL) {
  1269.             HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"error: adr is empty for back_add"LF);
  1270.     }
  1271.     return -1;    // erreur!
  1272.   }
  1273.   if (strnotempty(fil)==0) {
  1274.     if (opt->log!=NULL) {
  1275.       HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"error: fil is empty for back_add"LF);
  1276.     }
  1277.     return -1;    // erreur!
  1278.   }
  1279.   // FIN vΘrifier cohΘrence de adr et fil (non vide!)
  1280.  
  1281.   // stats
  1282.   opt->state.back_add_stats++;
  1283.  
  1284.   // rechercher emplacement
  1285.   back_clean(opt, cache, sback);
  1286.   if ( ( p = back_search(opt, sback) ) >= 0) {
  1287.     back[p].send_too[0]='\0';  // Θventuels paramΦtres supplΘmentaires α transmettre au serveur
  1288.  
  1289.     // clear r
  1290.     if (back[p].r.soc!=INVALID_SOCKET) {  /* we never know */
  1291.       deletehttp(&back[p].r);
  1292.     }
  1293.     memset(&(back[p].r), 0, sizeof(htsblk)); 
  1294.     back[p].r.soc=INVALID_SOCKET; 
  1295.     back[p].r.location=back[p].location_buffer;
  1296.  
  1297.     // crΘer entrΘe
  1298.     strcpybuff(back[p].url_adr,adr);
  1299.     strcpybuff(back[p].url_fil,fil);
  1300.     strcpybuff(back[p].url_sav,save);
  1301.     //back[p].links_index = links_index;
  1302.     // copier referer si besoin
  1303.     strcpybuff(back[p].referer_adr,"");
  1304.     strcpybuff(back[p].referer_fil,"");
  1305.     if ((referer_adr) && (referer_fil)) {       // existe
  1306.       if ((strnotempty(referer_adr)) && (strnotempty(referer_fil))) {   // non vide
  1307.         if (referer_adr[0]!='!') {    // non dΘtruit
  1308.           if (strcmp(referer_adr,"file://")) {      // PAS file://
  1309.             if (strcmp(referer_adr,"primary")) {      // pas referer 1er lien
  1310.               strcpybuff(back[p].referer_adr,referer_adr);
  1311.               strcpybuff(back[p].referer_fil,referer_fil);
  1312.             }
  1313.           }
  1314.         }
  1315.       }
  1316.     }
  1317.     // sav ne sert α rien pour le moment
  1318.     back[p].r.size=0;                   // rien n'a encore ΘtΘ chargΘ
  1319.     back[p].r.adr=NULL;                 // pas de bloc de mΘmoire
  1320.     back[p].r.is_write=0;               // α priori stockage en mΘmoire
  1321.     back[p].maxfile_html=opt->maxfile_html;
  1322.     back[p].maxfile_nonhtml=opt->maxfile_nonhtml;
  1323.     back[p].testmode=test;              // mode test?
  1324.     if (!opt->http10)                 // option "forcer 1.0" dΘsactivΘe
  1325.       back[p].http11=1;               // autoriser http/1.1
  1326.     back[p].head_request=0;
  1327.     if (strcmp(back[p].url_sav,BACK_ADD_TEST)==0)    // HEAD
  1328.       back[p].head_request=1;
  1329.     else if (strcmp(back[p].url_sav,BACK_ADD_TEST2)==0)    // test en GET
  1330.       back[p].head_request=2;       // test en get
  1331.  
  1332.     /* Stop requested - abort backing */
  1333.     if (opt->state.stop) {
  1334.       back[p].r.statuscode=STATUSCODE_INVALID;        // fatal
  1335.       strcpybuff(back[p].r.msg,"mirror stopped by user");
  1336.       back[p].status=STATUS_READY;  // terminΘ
  1337.       back_set_finished(sback, p);
  1338.       if ((opt->debug>0) && (opt->log!=NULL)) {
  1339.         HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"File not added due to mirror cancel: %s%s"LF,adr,fil); test_flush;
  1340.       }            
  1341.       return 0;
  1342.     }
  1343.  
  1344.     // test "fast header" cache ; that is, tests we did that lead to 3XX/4XX/5XX response codes
  1345.     if (cache->cached_tests != NULL) {
  1346.       intptr_t ptr = 0;
  1347.       if (inthash_read(cache->cached_tests, concat(OPT_GET_BUFF(opt), adr, fil), &ptr)) {    // gotcha
  1348.         if (ptr != 0) {
  1349.           char* text = (char*) ptr;
  1350.           char* lf = strchr(text, '\n');
  1351.           int code = 0;
  1352.           if (sscanf(text, "%d", &code) == 1) {     // got code
  1353.              back[p].r.statuscode=code;
  1354.              back[p].status=STATUS_READY;  // terminΘ
  1355.              if (lf != NULL && *lf != '\0') {     // got location ?
  1356.                strcpybuff(back[p].r.location, lf + 1);
  1357.              }
  1358.              return 0;
  1359.           }
  1360.         }
  1361.       }
  1362.     }
  1363.  
  1364.     // tester cache
  1365.     if ((strcmp(adr,"file://"))           /* pas fichier */
  1366.       && ( (!test) || (cache->type==1) )   /* cache prioritaire, laisser passer en test! */
  1367.       && ( (strnotempty(save)) || (strcmp(fil,"/robots.txt")==0) ) ) {  // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
  1368. #if HTS_FAST_CACHE
  1369.       intptr_t hash_pos;
  1370.       int hash_pos_return=0;
  1371. #else
  1372.       char* a=NULL;
  1373. #endif
  1374. #if HTS_FAST_CACHE
  1375.       if (cache->hashtable) { 
  1376. #else
  1377.       if (cache->use) { 
  1378. #endif
  1379.         char BIGSTK buff[HTS_URLMAXSIZE*4];
  1380. #if HTS_FAST_CACHE
  1381.         strcpybuff(buff,adr); strcatbuff(buff,fil);
  1382.         hash_pos_return=inthash_read(cache->hashtable,buff,&hash_pos);
  1383. #else
  1384.         buff[0]='\0'; strcatbuff(buff,"\n"); strcatbuff(buff,adr); strcatbuff(buff,"\n"); strcatbuff(buff,fil); strcatbuff(buff,"\n");
  1385.         a=strstr(cache->use,buff);
  1386. #endif
  1387.         
  1388.         // Ok, notΘ en cache->. mais bien prΘsent dans le cache ou sur disque?
  1389. #if HTS_FAST_CACHE
  1390.         if (hash_pos_return) {
  1391. #else
  1392.         if (a) {
  1393. #endif
  1394.           if (!test) {      // non mode test
  1395. #if HTS_FAST_CACHE
  1396.             uintptr_t pos=hash_pos;
  1397. #else
  1398.             int pos=-1;
  1399.             a+=strlen(buff);
  1400.             sscanf(a,"%d",&pos);    // lire position
  1401. #endif
  1402.             if (pos<0) {    // pas de mise en cache data, vΘrifier existence
  1403.               /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
  1404.               if (!IS_DELAYED_EXT(save) && fsize(fconv(catbuff,save)) <= 0) {  // fichier existe pas ou est vide!
  1405.                 int found=0;
  1406.  
  1407.                 /* It is possible that the file has been moved due to changes in build structure */
  1408.                 {
  1409.                   char BIGSTK previous_save[HTS_URLMAXSIZE*2];
  1410.                   previous_save[0] = '\0';
  1411.                   back[p].r = cache_readex(opt, cache, adr, fil, NULL, back[p].location_buffer, previous_save, 0);
  1412.                   if (previous_save[0] != '\0' && fexist(fconv(catbuff,previous_save))) {
  1413.                     rename(fconv(catbuff,previous_save), fconv(catbuff2,save));
  1414.                     if (fexist(fconv(catbuff,save))) {
  1415.                       found = 1;
  1416.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  1417.                         HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File '%s' has been renamed since last mirror to '%s' ; applying changes"LF, previous_save, save); test_flush;
  1418.                       }
  1419.                     } else {
  1420.                       if ((opt->debug>0) && (opt->log!=NULL)) {
  1421.                         HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"Could not rename '%s' to '%s' ; will have to retransfer it"LF, previous_save, save); test_flush;
  1422.                       }
  1423.                     }
  1424.                   }
  1425.                 }
  1426.                 
  1427.                 if (!found) {
  1428. #if HTS_FAST_CACHE
  1429.                   hash_pos_return=0;
  1430. #else
  1431.                   a=NULL;    
  1432. #endif
  1433.                   // dΘvalider car non prΘsent sur disque dans structure originale!!!
  1434.                   // sinon, le fichier est ok α priori, mais on renverra un if-modified-since pour
  1435.                   // en Ωtre s√r
  1436.                   if (opt->norecatch) {              // tester norecatch
  1437.                     if (!fexist(fconv(catbuff,save))) {  // fichier existe pas mais dΘclarΘ: on l'a effacΘ
  1438.                       FILE* fp=fopen(fconv(catbuff,save),"wb");
  1439.                       if (fp) fclose(fp);
  1440.                       if (opt->log!=NULL) {
  1441.                         HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"File must have been erased by user, ignoring: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  1442.                       }
  1443.                     }
  1444.                   }
  1445.                 }
  1446.               }  // fsize() <= 0
  1447.             }
  1448.           }
  1449.         }
  1450.         //
  1451.       } else
  1452. #if HTS_FAST_CACHE
  1453.         hash_pos_return=0;
  1454. #else
  1455.         a=NULL;
  1456. #endif
  1457.  
  1458.       // Existe pas en cache, ou bien pas de cache prΘsent
  1459. #if HTS_FAST_CACHE
  1460.       if (hash_pos_return) {  // OK existe en cache (et donnΘes aussi)!
  1461. #else
  1462.       if (a!=NULL) {  // OK existe en cache (et donnΘes aussi)!
  1463. #endif
  1464.         if (cache->type==1) {   // cache prioritaire (pas de test if-modified..)
  1465.                                // dans ce cas on peut Θgalement lire des rΘponses cachΘes comme 404,302...
  1466.           // lire dans le cache
  1467.           if (!test)
  1468.             back[p].r = cache_read(opt,cache,adr,fil,save, back[p].location_buffer);
  1469.           else
  1470.             back[p].r = cache_read(opt,cache,adr,fil,NULL, back[p].location_buffer);  // charger en tΩte uniquement du cache
  1471.  
  1472.           /* ensure correct location buffer set */
  1473.           back[p].r.location=back[p].location_buffer;
  1474.  
  1475.           /* Interdiction taille par le wizard? --> dΘtruire */
  1476.           if (back[p].r.statuscode != -1) {  // pas d'erreur de lecture
  1477.             if (!back_checksize(opt,&back[p],0)) {
  1478.               back[p].status=STATUS_READY;  // FINI
  1479.               back_set_finished(sback, p);
  1480.               back[p].r.statuscode=STATUSCODE_TOO_BIG;
  1481.               if (!back[p].testmode)
  1482.                 strcpybuff(back[p].r.msg,"Cached file skipped (too big)");
  1483.               else
  1484.                 strcpybuff(back[p].r.msg,"Test: Cached file skipped  (too big)");
  1485.               return 0;
  1486.             }
  1487.           }
  1488.  
  1489.           if (back[p].r.statuscode != -1 || IS_DELAYED_EXT(save)) {  // pas d'erreur de lecture ou test retardΘ
  1490.             if ((opt->debug>0) && (opt->log!=NULL)) {
  1491.               if (!test) {
  1492.                 HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File immediately loaded from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  1493.               } else {
  1494.                 HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File immediately tested from cache: %s%s"LF,back[p].url_adr,back[p].url_fil); test_flush;
  1495.               }
  1496.             }
  1497.             back[p].r.notmodified=1;    // fichier non modifiΘ
  1498.             back[p].status=STATUS_READY;  // OK prΩt
  1499.             //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
  1500.             back_set_finished(sback, p);
  1501.  
  1502.             // finalize transfer
  1503.             if (!test) {
  1504.               if (back[p].r.statuscode>0) {
  1505.                 back_finalize(opt,cache,sback,p);
  1506.               }
  1507.             }
  1508.  
  1509.             return 0;
  1510.           } else {  // erreur
  1511.             // effacer r
  1512.             memset(&(back[p].r), 0, sizeof(htsblk)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  1513.             // et continuer (chercher le fichier)
  1514.           }
  1515.           
  1516.         } else if (cache->type==2) {    // si en cache, demander de tester If-Modified-Since
  1517.           htsblk r;
  1518.           cache_header(opt,cache,adr,fil,&r);
  1519.  
  1520.           /* Interdiction taille par le wizard? */
  1521.           {
  1522.             LLint save_totalsize=back[p].r.totalsize;
  1523.             back[p].r.totalsize=r.totalsize;
  1524.             if (!back_checksize(opt,&back[p],1)) {
  1525.               r.statuscode = STATUSCODE_INVALID;
  1526.               //
  1527.               back[p].status=STATUS_READY;  // FINI
  1528.               back_set_finished(sback, p);
  1529.               back[p].r.statuscode=STATUSCODE_TOO_BIG;
  1530.               deletehttp(&back[p].r); back[p].r.soc=INVALID_SOCKET;
  1531.               if (!back[p].testmode)
  1532.                 strcpybuff(back[p].r.msg,"File too big");
  1533.               else
  1534.                 strcpybuff(back[p].r.msg,"Test: File too big");
  1535.               return 0;
  1536.             }
  1537.             back[p].r.totalsize=save_totalsize;
  1538.           }
  1539.           
  1540.           if (r.statuscode != -1) {
  1541.             if (r.statuscode==HTTP_OK) {     // uniquement des 200 (OK)
  1542.               if (strnotempty(r.etag)) {  // ETag (RFC2616)
  1543.                 /*
  1544.                 - If both an entity tag and a Last-Modified value have been
  1545.                 provided by the origin server, SHOULD use both validators in
  1546.                 cache-conditional requests. This allows both HTTP/1.0 and
  1547.                 HTTP/1.1 caches to respond appropriately.
  1548.                 */
  1549.                 if (strnotempty(r.lastmodified))
  1550.                   sprintf(back[p].send_too,"If-None-Match: %s\r\nIf-Modified-Since: %s\r\n",r.etag,r.lastmodified);
  1551.                 else
  1552.                   sprintf(back[p].send_too,"If-None-Match: %s\r\n",r.etag);
  1553.               }
  1554.               else if (strnotempty(r.lastmodified))
  1555.                 sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",r.lastmodified);
  1556.               else if (strnotempty(cache->lastmodified))
  1557.                 sprintf(back[p].send_too,"If-Modified-Since: %s\r\n",cache->lastmodified);
  1558.               
  1559.               /* this is an update of a file */
  1560.               if (strnotempty(back[p].send_too))
  1561.                 back[p].is_update=1;
  1562.               back[p].r.req.nocompression=1;   /* Do not compress when updating! */
  1563.               
  1564.             }
  1565.           }
  1566. #if DEBUGCA
  1567.           printf("..is modified test %s\n",back[p].send_too);
  1568. #endif
  1569.         } 
  1570.         // Okay, pas trouvΘ dans le cache
  1571.         // Et si le fichier existe sur disque?
  1572.         // Pas dans le cache: fichier n'a pas ΘtΘ transfΘrΘ du tout, donc pas sur disque?
  1573.       } else {
  1574.         if (fexist(save)) {    // fichier existe? aghl!
  1575.           off_t sz=fsize(save);
  1576.           // Bon, lα il est possible que le fichier ait ΘtΘ partiellement transfΘrΘ
  1577.           // (s'il l'avait ΘtΘ en totalitΘ il aurait ΘtΘ inscrit dans le cache ET existerait sur disque)
  1578.           // PAS de If-Modified-Since, on a pas connaissance des donnΘes α la date du cache
  1579.           // On demande juste les donnΘes restantes si le date est valide (206), tout sinon (200)
  1580.           if ((ishtml(opt,save) != 1) && (ishtml(opt,back[p].url_fil)!=1)) {   // NON HTML (liens changΘs!!)
  1581.             if (sz>0) {    // Fichier non vide? (question bΩte, sinon on transfert tout!)
  1582.               char lastmodified[256];
  1583.               get_filetime_rfc822(save, lastmodified);
  1584.               if (strnotempty(lastmodified)) {     /* pas de If-.. possible */
  1585. #if DEBUGCA
  1586.                 printf("..if unmodified since %s size "LLintP"\n", lastmodified, (LLint)sz);
  1587. #endif
  1588.                 if ((opt->debug>1) && (opt->log!=NULL)) {
  1589.                   HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File partially present ("LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush;
  1590.                 }
  1591.                 
  1592.                 /* impossible - don't have etag or date
  1593.                 if (strnotempty(back[p].r.etag)) {  // ETag (RFC2616)
  1594.                 sprintf(back[p].send_too,"If-None-Match: %s\r\n",back[p].r.etag);
  1595.                 back[p].http11=1;    // En tΩte 1.1
  1596.                 } else if (strnotempty(back[p].r.lastmodified)) {
  1597.                 sprintf(back[p].send_too,"If-Unmodified-Since: %s\r\n",back[p].r.lastmodified);
  1598.                 back[p].http11=1;    // En tΩte 1.1
  1599.                 } else 
  1600.                 */
  1601.                 if (strlen(lastmodified)) {
  1602.                   sprintf(back[p].send_too,
  1603.                     "If-Unmodified-Since: %s\r\nRange: bytes="LLintP"-\r\n"
  1604.                     , lastmodified, (LLint)sz);
  1605.                   back[p].http11=1;    // En tΩte 1.1
  1606.                   back[p].range_req_size=sz;
  1607.                   back[p].r.req.range_used=1;
  1608.                   back[p].r.req.nocompression=1;
  1609.                 } else {
  1610.                   if ((opt->debug>0) && (opt->log!=NULL)) {
  1611.                     HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Could not find timestamp for partially present file, restarting (lost "LLintP" bytes): %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); test_flush;
  1612.                   }
  1613.                 }
  1614.                 
  1615.               } else { 
  1616.                 if ((opt->debug>0) && (opt->log!=NULL)) {
  1617.                   HTS_LOG(opt,LOG_WARNING);
  1618.                   /*
  1619.                   if (opt->http10)
  1620.                   fprintf(opt->log,"File partially present (%d bytes) retransfered due to HTTP/1.0 settings: %s%s"LF,sz,back[p].url_adr,back[p].url_fil);
  1621.                   else
  1622.                   */
  1623.                   fprintf(opt->log,"File partially present ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); 
  1624.                   test_flush;
  1625.                 }
  1626.                 /* Sinon requΩte normale... */
  1627.                 back[p].http11=0;
  1628.               }
  1629.             } else if (opt->norecatch) {              // tester norecatch
  1630.               filenote(&opt->state.strc,save,NULL);       // ne pas purger tout de mΩme
  1631.               file_notify(opt,back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);
  1632.               back[p].status=STATUS_READY;  // OK prΩt
  1633.               back_set_finished(sback, p);
  1634.               back[p].r.statuscode=STATUSCODE_INVALID;  // erreur
  1635.               strcpybuff(back[p].r.msg,"Null-size file not recaught");
  1636.               return 0;
  1637.             }
  1638.           } else {
  1639.             if ((opt->debug>0) && (opt->log!=NULL)) {
  1640.               HTS_LOG(opt,LOG_WARNING);
  1641.               fprintf(opt->log,"HTML file ("LLintP" bytes) retransfered due to lack of cache: %s%s"LF,(LLint)sz,back[p].url_adr,back[p].url_fil); 
  1642.               test_flush;
  1643.             }
  1644.             /* Sinon requΩte normale... */
  1645.             back[p].http11=0;
  1646.           }
  1647.         }
  1648.       }
  1649.     }
  1650.  
  1651.  
  1652.     {
  1653.       ///htsblk r;   non directement dans la structure-rΘponse!
  1654.       T_SOC soc;
  1655.       
  1656.       // ouvrir liaison, envoyer requΦte
  1657.       // ne pas traiter ou recevoir l'en tΩte immΘdiatement
  1658.       memset(&(back[p].r), 0, sizeof(htsblk)); back[p].r.soc=INVALID_SOCKET; back[p].r.location=back[p].location_buffer;
  1659.       // recopier proxy
  1660.       if ((back[p].r.req.proxy.active = opt->proxy.active)) {
  1661.         if (StringBuff(opt->proxy.bindhost) != NULL)
  1662.           strcpybuff(back[p].r.req.proxy.bindhost, StringBuff(opt->proxy.bindhost));
  1663.         if (StringBuff(opt->proxy.name) != NULL)
  1664.           strcpybuff(back[p].r.req.proxy.name, StringBuff(opt->proxy.name));
  1665.         back[p].r.req.proxy.port = opt->proxy.port;
  1666.       }
  1667.       // et user-agent
  1668.       strcpy(back[p].r.req.user_agent,StringBuff(opt->user_agent));
  1669.       strcpy(back[p].r.req.referer,StringBuff(opt->referer));
  1670.       strcpy(back[p].r.req.from,StringBuff(opt->from));
  1671.       strcpy(back[p].r.req.lang_iso,StringBuff(opt->lang_iso));
  1672.       back[p].r.req.user_agent_send=opt->user_agent_send;
  1673.       // et http11
  1674.       back[p].r.req.http11=back[p].http11;
  1675.       back[p].r.req.nocompression=opt->nocompression;
  1676.       back[p].r.req.nokeepalive=opt->nokeepalive;
  1677.  
  1678.       // mode ftp, court-circuit!
  1679.       if (strfield(back[p].url_adr,"ftp://")) {
  1680.         if (back[p].testmode) {
  1681.                 if (opt->log!=NULL) {
  1682.             HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"error: forbidden test with ftp link for back_add"LF);
  1683.           }
  1684.           return -1;    // erreur pas de test permis
  1685.         }
  1686.         if (!(back[p].r.req.proxy.active && opt->ftp_proxy)) { // connexion directe, gΘrΘe en thread
  1687.                     FTPDownloadStruct *str = (FTPDownloadStruct*) malloc(sizeof(FTPDownloadStruct));
  1688.                     str->pBack = &back[p];
  1689.                     str->pOpt = opt;
  1690.                     /* */
  1691.           back[p].status=STATUS_FTP_TRANSFER;   // connexion ftp
  1692. #if USE_BEGINTHREAD
  1693.                     launch_ftp(str);
  1694. #else
  1695. #error Must have pthreads
  1696. #endif
  1697.           return 0;
  1698.         }
  1699.       }
  1700. #if HTS_USEMMS
  1701.       else if (strfield(back[p].url_adr,"mms://")) {
  1702.                 MMSDownloadStruct str;
  1703.         if (back[p].testmode) {
  1704.                 if (opt->log!=NULL) {
  1705.             HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"error: forbidden test with mms link for back_add"LF);
  1706.           }
  1707.           return -1;    // erreur pas de test permis
  1708.         }
  1709.         if (back[p].r.req.proxy.active) {
  1710.           HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"warning: direct connection for mms links (proxy settings ignored)"LF);
  1711.                 }
  1712.                 back[p].status=STATUS_FTP_TRANSFER;   // connexion externe
  1713.                 str.pBack = &back[p];
  1714.                 str.pOpt = opt;
  1715.                 launch_mms(&str);
  1716.                 return 0;
  1717.       }
  1718. #endif
  1719. #if HTS_USEOPENSSL
  1720.       else if (SSL_is_available && strfield(back[p].url_adr,"https://")) {        // let's rock
  1721.         back[p].r.ssl = 1;
  1722.         // back[p].r.ssl_soc = NULL;
  1723.         back[p].r.ssl_con = NULL;
  1724.       }
  1725. #endif
  1726.       
  1727.       if (!back_trylive(opt, cache, sback, p)) {
  1728. #if HTS_XGETHOST
  1729. #if HDEBUG
  1730.         printf("back_solve..\n");
  1731. #endif
  1732.         back[p].status=STATUS_WAIT_DNS;    // tentative de rΘsolution du nom de host
  1733.         soc=INVALID_SOCKET;    // pas encore ouverte
  1734.         back_solve(opt, &back[p]);  // prΘparer
  1735.         if (host_wait(opt, &back[p])) {  // prΩt, par ex fichier ou dispo dans dns
  1736. #if HDEBUG
  1737.           printf("ok, dns cache ready..\n");
  1738. #endif
  1739.           soc=http_xfopen(opt,0,0,0,back[p].send_too,adr,fil,&(back[p].r));
  1740.           if (soc==INVALID_SOCKET) {
  1741.             back[p].status=STATUS_READY;  // fini, erreur
  1742.             back_set_finished(sback, p);
  1743.           }
  1744.         }
  1745.         //
  1746. #else
  1747.         //
  1748. #if CNXDEBUG
  1749.         printf("XFopen..\n");
  1750. #endif
  1751.         
  1752.         if (strnotempty(back[p].send_too))    // envoyer un if-modified-since
  1753. #if HTS_XCONN
  1754.           soc=http_xfopen(0,0,0,back[p].send_too,adr,fil,&(back[p].r));
  1755. #else
  1756.         soc=http_xfopen(0,0,1,back[p].send_too,adr,fil,&(back[p].r));
  1757. #endif
  1758.         else
  1759. #if HTS_XCONN
  1760.           soc=http_xfopen(test,0,0,NULL,adr,fil,&(back[p].r));
  1761. #else
  1762.         soc=http_xfopen(test,0,1,NULL,adr,fil,&(back[p].r));
  1763. #endif
  1764. #endif
  1765.       } else {
  1766.         soc = back[p].r.soc;
  1767.  
  1768.         if ((opt->debug>1) && (opt->log!=NULL)) {
  1769.           HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"(Keep-Alive): successfully linked #%d (for %s%s)"LF, 
  1770.             back[p].r.debugid,
  1771.             back[p].url_adr, back[p].url_fil); test_flush;
  1772.         }
  1773.       }
  1774.       
  1775.       if (opt->timeout>0) {    // gestion du opt->timeout
  1776.         back[p].timeout=opt->timeout;
  1777.         back[p].timeout_refresh=time_local();
  1778.       } else {
  1779.         back[p].timeout=-1;    // pas de gestion (default)
  1780.       }
  1781.       
  1782.       if (opt->rateout>0) {    // gestion d'un taux minimum de transfert tolΘrΘ
  1783.         back[p].rateout=opt->rateout;
  1784.         back[p].rateout_time=time_local();
  1785.       } else {
  1786.         back[p].rateout=-1;    // pas de gestion (default)
  1787.       }
  1788.  
  1789.       // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas o∙ cela est
  1790.       // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
  1791.       //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
  1792.  
  1793. #if CNXDEBUG
  1794. printf("Xfopen ok, poll..\n");
  1795. #endif
  1796.  
  1797. #if HTS_XGETHOST
  1798.     if (soc!=INVALID_SOCKET)
  1799.       if (back[p].status==STATUS_WAIT_DNS) {  // pas d'erreur
  1800.         if (!back[p].r.is_file)
  1801.           back[p].status=STATUS_CONNECTING;   // connexion en cours
  1802.         else
  1803.           back[p].status=1;     // fichier
  1804.       }
  1805.  
  1806. #else
  1807.       if (soc==INVALID_SOCKET) { // erreur socket
  1808.         back[p].status=STATUS_READY;    // FINI
  1809.         back_set_finished(sback, p);
  1810.         //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
  1811.         back[p].r.soc=INVALID_SOCKET;
  1812.       } else {
  1813.         if (!back[p].r.is_file)
  1814. #if HTS_XCONN
  1815.           back[p].status=STATUS_CONNECTING;   // connexion en cours
  1816. #else
  1817.           back[p].status=99;    // chargement en tΩte en cours
  1818. #endif
  1819.         else
  1820.           back[p].status=1;     // chargement fichier
  1821. #if BDEBUG==1
  1822.         printf("..loading header\n");
  1823. #endif
  1824.       }
  1825. #endif
  1826.       
  1827.     }
  1828.  
  1829.  
  1830.     // note: si il y a erreur (404,etc) status=2 (terminΘ/Θchec) mais
  1831.     // le lien est considΘrΘ comme traitΘ
  1832.     //if (back[p].soc<0)  // erreur
  1833.     //  return -1;
  1834.  
  1835.     return 0;
  1836.   } else {
  1837.     if (opt->log!=NULL) {
  1838.       HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"error: no space left in stack for back_add"LF);
  1839.       if ( ( opt->state.debug_state & 1 ) == 0 ) {    /* debug_state<0> == debug 'no space left in stack' */
  1840.         int i;
  1841.         HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"debug: DUMPING %d BLOCKS"LF, back_max);
  1842.         opt->state.debug_state |= 1;    /* once */
  1843.         /* OUTPUT FULL DEBUG INFORMATION THE FIRST TIME WE SEE THIS VERY ANNOYING BUG,
  1844.         HOPING THAT SOME USER REPORT WILL QUICKLY SOLVE THIS PROBLEM :p */
  1845.         for(i = 0 ; i < back_max ; i++ ) {
  1846.           if (back[i].status != -1) {
  1847.             int may_clean = slot_can_be_cleaned(&back[i]);
  1848.             int may_finalize = may_clean && slot_can_be_finalized(opt, &back[i]);
  1849.             int may_serialize = slot_can_be_cached_on_disk(&back[i]);
  1850.             HTS_LOG(opt,LOG_INFO);
  1851.             fprintf(opt->log,
  1852.               "debug: back[%03d]: may_clean=%d, may_finalize_disk=%d, may_serialize=%d:"LF
  1853.               "\t" "finalized(%d), status(%d), locked(%d), delayed(%d), test(%d), "LF
  1854.               "\t" "statuscode(%d), size(%d), is_write(%d), may_hypertext(%d), "LF
  1855.               "\t" "contenttype(%s), url(%s%s), save(%s)"LF,
  1856.               i,
  1857.               may_clean, may_finalize, may_serialize,
  1858.               back[i].finalized,
  1859.               back[i].status,
  1860.               back[i].locked,
  1861.               IS_DELAYED_EXT(back[i].url_sav),
  1862.               back[i].testmode,
  1863.               back[i].r.statuscode,
  1864.               (int) back[i].r.size,
  1865.               back[i].r.is_write,
  1866.               may_be_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil),
  1867.               /* */
  1868.               back[i].r.contenttype,
  1869.               back[i].url_adr, back[i].url_fil,
  1870.               back[i].url_sav ? back[i].url_sav : "<null>"
  1871.               );
  1872.           }
  1873.         }
  1874.       }
  1875.  
  1876.     }
  1877.     return -1;    // plus de place
  1878.   }
  1879. }
  1880.  
  1881.  
  1882.  
  1883. #if HTS_XGETHOST
  1884. #if USE_BEGINTHREAD
  1885. // lancement multithread du robot
  1886. typedef struct {
  1887.     char iadr_p[HTS_URLMAXSIZE];
  1888.     httrackp *opt;
  1889. } HostlookupStruct;
  1890. void Hostlookup(void* pP) {
  1891.     HostlookupStruct *str = (HostlookupStruct*) pP;
  1892.   char iadr[256];
  1893.     t_dnscache* cache=_hts_cache(str->opt);  // adresse du cache
  1894.   t_hostent* hp;
  1895.   int error_found=0;
  1896.  
  1897.   // recopier (aprΦs id:pass)
  1898. #if DEBUGDNS 
  1899.   printf("resolv in background: %s\n",jump_identification(iadr_p));
  1900. #endif
  1901.   strcpybuff(iadr,jump_identification(str->iadr_p));
  1902.   // couper Θventuel :
  1903.   {
  1904.     char *a;
  1905.     if ( (a=jump_toport(iadr)) )
  1906.       *a='\0';          // get rid of it
  1907.   }
  1908.   freet(pP);
  1909.  
  1910.   hts_mutexlock(&dns_lock);
  1911.  
  1912.   while(cache->n) {
  1913.     if (strcmp(cache->iadr,iadr)==0) {
  1914.       error_found=1;
  1915.     }
  1916.     cache=cache->n;    // calculer queue
  1917.   }
  1918.   if (strcmp(cache->iadr,iadr)==0) {
  1919.     error_found=1;
  1920.   }
  1921.  
  1922.   if (!error_found) {
  1923.     // en gros copie de hts_gethostbyname sans le return
  1924.     cache->n=(t_dnscache*) calloct(1,sizeof(t_dnscache));
  1925.     if (cache->n!=NULL) {
  1926.       t_fullhostent fullhostent_buffer;
  1927.       strcpybuff(cache->n->iadr,iadr);
  1928.       cache->n->host_length=0;        /* pour le moment rien */
  1929.       cache->n->n=NULL;
  1930.       
  1931.       /* resolve */
  1932. #if DEBUGDNS 
  1933.       printf("gethostbyname() in progress for %s\n",iadr);
  1934. #endif
  1935.       cache->n->host_length=-1;
  1936.       memset(cache->n->host_addr, 0, sizeof(cache->n->host_addr));
  1937.       hp=vxgethostbyname(iadr, &fullhostent_buffer);
  1938.       if (hp!=NULL) {
  1939.         memcpy(cache->n->host_addr, hp->h_addr, hp->h_length);
  1940.         cache->n->host_length = hp->h_length;
  1941.       }
  1942.     }
  1943.   } else {
  1944. #if DEBUGDNS 
  1945.     printf("aborting resolv for %s (found)\n",iadr);
  1946. #endif
  1947.   }
  1948.  
  1949.   hts_mutexrelease(&dns_lock);
  1950.  
  1951. #if DEBUGDNS 
  1952.   printf("quitting resolv for %s (result: %d)\n",iadr,(cache->n!=NULL)?cache->n->host_length:(-999));
  1953. #endif
  1954. }
  1955. #endif
  1956.  
  1957. // attendre que le host (ou celui du proxy) ait ΘtΘ rΘsolu
  1958. // si c'est un fichier, la rΘsolution est immΘdiate
  1959. // idem pour ftp://
  1960. void back_solve(httrackp *opt, lien_back* back) {
  1961.   if ((!strfield(back->url_adr,"file://")) 
  1962.         && ! strfield(back->url_adr,"ftp://")
  1963. #if HTS_USEMMS
  1964.         && ! strfield(back->url_adr,"mms://")
  1965. #endif
  1966.         ) {
  1967.   //## if (back->url_adr[0]!=lOCAL_CHAR) {  // qq chose α prΘparer
  1968.     const char* a;
  1969.     if (!(back->r.req.proxy.active))
  1970.       a=back->url_adr;
  1971.     else
  1972.       a=back->r.req.proxy.name;
  1973.     a = jump_protocol(a);
  1974.     if (!hts_dnstest(opt, a)) {   // non encore testΘ!..
  1975.       // inscire en thread
  1976. #ifdef _WIN32
  1977.       // Windows
  1978. #if USE_BEGINTHREAD
  1979.       {
  1980.                 HostlookupStruct *str = (HostlookupStruct*)malloct(sizeof(HostlookupStruct));
  1981.         if (str) {
  1982.                     strcpybuff(str->iadr_p, a);
  1983.                     str->opt = opt;
  1984.           hts_newthread(Hostlookup, str);
  1985.         }
  1986.       }
  1987. #else
  1988.       /*t_hostent* h=*/
  1989.       /*hts_gethostbyname(a);*/  // calcul
  1990. #endif
  1991. #else
  1992. #if USE_BEGINTHREAD
  1993.         char* p = calloct(strlen(a)+2,1);
  1994.         if (p) {
  1995.           strcpybuff(p,a);
  1996.           hts_newthread( Hostlookup , p );
  1997.         }
  1998. #else
  1999.       // Sous Unix, le gethostbyname() est bloquant..
  2000.       /*t_hostent* h=*/
  2001.       /*hts_gethostbyname(a);*/  // calcul
  2002. #endif
  2003. #endif
  2004.     }
  2005.   }
  2006. }
  2007.  
  2008. // dΘtermine si le host a pu Ωtre rΘsolu
  2009. int host_wait(httrackp *opt, lien_back* back) {
  2010.   if ((!strfield(back->url_adr,"file://")) 
  2011.         && (!strfield(back->url_adr,"ftp://"))
  2012. #if HTS_USEMMS
  2013.         && (!strfield(back->url_adr,"mms://"))
  2014. #endif
  2015.         ) {
  2016.   //## if (back->url_adr[0]!=lOCAL_CHAR) {
  2017.     if (!(back->r.req.proxy.active)) {
  2018.       return (hts_dnstest(opt, back->url_adr));
  2019.     } else {
  2020.       return (hts_dnstest(opt, back->r.req.proxy.name));      
  2021.     }
  2022.   } else return 1;    // prΩt, fichier local
  2023. }
  2024. #endif
  2025.  
  2026.  
  2027. // Θlimine les fichiers non html en backing (anticipation)
  2028. // cleanup non-html files in backing to save backing space
  2029. // and allow faster "save in cache" operation
  2030. // also cleanup keep-alive sockets and ensure that not too many sockets are being opened
  2031.  
  2032. static int slot_can_be_cleaned(const lien_back* back) {
  2033.   return 
  2034.     (back->status == STATUS_READY)                        // ready
  2035.     /* Check autoclean */
  2036.     && (!back->testmode)                                  // not test mode
  2037.     && (strnotempty(back->url_sav))                       // filename exists
  2038.     && (HTTP_IS_OK(back->r.statuscode))                   // HTTP "OK"
  2039.     && (back->r.size > 0)                                 // size>0
  2040.     ;
  2041. }
  2042.  
  2043. static int slot_can_be_finalized(httrackp* opt, const lien_back* back) {
  2044.   return
  2045.     (back->r.is_write                             // not in memory (on disk, ready)
  2046.     && !is_hypertext_mime(opt,back->r.contenttype, back->url_fil)        // not HTML/hypertext
  2047.     && !may_be_hypertext_mime(opt,back->r.contenttype, back->url_fil)    // may NOT be parseable mime type
  2048.     );
  2049. }
  2050.  
  2051. void back_clean(httrackp* opt,cache_back* cache,struct_back* sback) {
  2052.   lien_back* const back = sback->lnk;
  2053.   const int back_max = sback->count;
  2054.   int oneMore = ( (opt->state._hts_in_html_parsing == 2 && opt->maxsoc >= 2) || (opt->state._hts_in_html_parsing == 1 && opt->maxsoc >= 4) ) ? 1 : 0;  // testing links
  2055.   int i;
  2056.   for(i=0;i<back_max;i++) {
  2057.     if (slot_can_be_cleaned(&back[i])) {
  2058.       if (slot_can_be_finalized(opt, &back[i])) {
  2059.         (void) back_flush_output(opt, cache, sback, i);        // flush output buffers
  2060.         usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr, back[i].url_fil);
  2061.         //if (back[i].links_index >= 0) {
  2062.         //  assertf(back[i].links_index < opt->hash->max_lien);
  2063.         //  opt->hash->liens[back[i].links_index]->pass2 = -1;
  2064.         //  // *back[i].pass2_ptr=-1;  // Done!
  2065.         //}
  2066.         /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
  2067.         {
  2068.           int index = hash_read(opt->hash,back[i].url_sav,"",0,opt->urlhack);      // lecture type 0 (sav)
  2069.           if (index >= 0) {
  2070.             opt->hash->liens[index]->pass2 = -1;      /* DONE! */
  2071.           } else {
  2072.             if (opt->log != NULL) {
  2073.               HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"engine: warning: entry cleaned up, but no trace on heap: %s%s (%s)"LF,back[i].url_adr, back[i].url_fil,back[i].url_sav);
  2074.               test_flush;
  2075.             }
  2076.           }
  2077.         }
  2078.         HTS_STAT.stat_background++;
  2079.         if ((opt->debug>0) && (opt->log!=NULL)) {
  2080.           HTS_LOG(opt,LOG_INFO); fprintf(opt->log,"File successfully written in background: %s"LF,back[i].url_sav); test_flush;
  2081.         }
  2082.         back_maydelete(opt,cache,sback,i);    // May delete backing entry
  2083.       } else {
  2084.         if (!back[i].finalized) {
  2085.           if (1) {
  2086.             /* Ensure deleted or recycled socket */
  2087.             /* BUT DO NOT YET WIPE back[i].r.adr */
  2088.             if ( (opt->debug>1) && (opt->log!=NULL) ) {
  2089.               HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"file %s%s validated (cached, left in memory)"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2090.             }
  2091.             back_maydeletehttp(opt, cache, sback, i);
  2092.           } else {
  2093.             /*
  2094.             NOT YET HANDLED CORRECTLY (READ IN NEW CACHE TO DO)
  2095.             */
  2096.             /* Lock the entry but do not keep the html data in memory (in cache) */
  2097.             if (opt->cache) {
  2098.               htsblk r;
  2099.  
  2100.               /* Ensure deleted or recycled socket */
  2101.               back_maydeletehttp(opt, cache, sback, i);
  2102.               assertf(back[i].r.soc == INVALID_SOCKET);
  2103.  
  2104.               /* Check header */
  2105.               cache_header(opt,cache,back[i].url_adr,back[i].url_fil,&r);
  2106.               if (r.statuscode == HTTP_OK) {
  2107.                 if (back[i].r.soc == INVALID_SOCKET) {
  2108.                   /* Delete buffer and sockets */
  2109.                   deleteaddr(&back[i].r);
  2110.                   deletehttp(&back[i].r);
  2111.                   if ( (opt->debug>1) && (opt->log!=NULL) ) {
  2112.                     HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"file %s%s temporarily left in cache to spare memory"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2113.                   }
  2114.                 }
  2115.               } else {
  2116.                 if ((opt->debug>0) && (opt->log!=NULL)) {
  2117.                   HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Unexpected html cache lookup error during back clean"LF); test_flush;
  2118.                 }            
  2119.               }
  2120.               // xxc xxc
  2121.             }
  2122.           }
  2123.         }
  2124.       }
  2125.     } else if (back[i].status == STATUS_ALIVE) {                         // waiting (keep-alive)
  2126.       if (
  2127.         ! back[i].r.keep_alive
  2128.         || back[i].r.soc == INVALID_SOCKET
  2129.         || back[i].r.keep_alive_max < 1
  2130.         || time_local() >= back[i].ka_time_start + back[i].r.keep_alive_t
  2131.         ) {
  2132.         if ((opt->debug>0) && (opt->log!=NULL)) {
  2133.             HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"(Keep-Alive): live socket closed #%d (%s)"LF, 
  2134.             back[i].r.debugid,
  2135.             back[i].url_adr);
  2136.             test_flush;
  2137.         }
  2138.         back_delete(opt,cache,sback, i);    // delete backing entry
  2139.       }
  2140.     }
  2141.   }
  2142.   /* switch connections to live ones */
  2143.   for(i=0;i<back_max;i++) {
  2144.     if (back[i].status == STATUS_READY) {                                   // ready
  2145.       if (back[i].r.soc != INVALID_SOCKET) {
  2146.         back_maydeletehttp(opt,cache,sback, i);
  2147.       }
  2148.     }
  2149.   }
  2150.   /* delete sockets if too many keep-alive'd sockets in background */
  2151.   if (opt->maxsoc > 0) {
  2152.     int max = opt->maxsoc + oneMore;
  2153.     int curr = back_nsoc_overall(sback);
  2154.     if (curr > max) {
  2155.       if ((opt->debug>1) && (opt->log!=NULL)) {
  2156.         HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"(Keep-Alive): deleting #%d sockets"LF, 
  2157.           curr - max); test_flush;
  2158.       }
  2159.     }
  2160.     for(i = 0 ; i < back_max && curr > max ; i++) {
  2161.       if (back[i].status == STATUS_ALIVE) {
  2162.         back_delete(opt,cache,sback, i);    // delete backing entry
  2163.         curr--;
  2164.       }
  2165.     }
  2166.   }
  2167.   /* transfer ready slots to the storage hashtable */
  2168.   {
  2169.     int nxfr = back_cleanup_background(opt,cache,sback);
  2170.     if (nxfr > 0 && (opt->debug>0) && (opt->log!=NULL)) {
  2171.       HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"(htsback): %d slots ready moved to background"LF, nxfr);
  2172.       test_flush;
  2173.     }
  2174.   }
  2175. }
  2176.  
  2177.  
  2178. // attente (gestion des buffers des sockets)
  2179. void back_wait(struct_back* sback,httrackp* opt,cache_back* cache,TStamp stat_timestart) {
  2180.     char catbuff[CATBUFF_SIZE];
  2181.   lien_back* const back = sback->lnk;
  2182.   const int back_max = sback->count;
  2183.   unsigned int i_mod;
  2184.   T_SOC nfds=INVALID_SOCKET;
  2185.   fd_set fds,fds_c,fds_e;     // fds pour lecture, connect (write), et erreur
  2186.   int nsockets;     // nbre sockets
  2187.   LLint max_read_bytes;  // max bytes read per sockets
  2188.   struct timeval tv;
  2189.   int do_wait=0;
  2190.   int gestion_timeout=0;
  2191.   int busy_recv=0;     // pas de donnΘes pour le moment   
  2192.   int busy_state=0;    // pas de connexions
  2193.   int max_loop;  // nombre de boucles max α parcourir..
  2194.   int max_loop_chk=0;
  2195.   unsigned int mod_random = (unsigned int) ( time_local() + HTS_STAT.HTS_TOTAL_RECV );
  2196.  
  2197.   // max. number of loops
  2198.   max_loop=8;
  2199.  
  2200. #if 1
  2201.   // Cleanup the stack to save space!
  2202.   back_clean(opt,cache,sback);
  2203. #endif
  2204.  
  2205.   // recevoir tant qu'il y a des donnΘes (avec un maximum de max_loop boucles)
  2206.   do_wait=0;
  2207.   gestion_timeout=0;
  2208.   do {
  2209.     int max_c;
  2210.     busy_state=busy_recv=0;
  2211.  
  2212. #if 0
  2213.     check_rate(stat_timestart,opt->maxrate);    // vΘrifier taux de transfert
  2214. #endif
  2215.     // inscrire les sockets actuelles, et rechercher l'ID la plus ΘlevΘe
  2216.     FD_ZERO(&fds);
  2217.     FD_ZERO(&fds_c);
  2218.     FD_ZERO(&fds_e);
  2219.     nsockets=0;
  2220.     max_read_bytes=TAILLE_BUFFER;     // maximum bytes that can be read
  2221.     nfds=INVALID_SOCKET;
  2222.  
  2223.     max_c=1;
  2224.     for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  2225.     // for(i=0;i<back_max;i++) {
  2226.         unsigned int i = ( i_mod + mod_random ) % ( back_max );
  2227.  
  2228.       // en cas de gestion du connect prΘemptif
  2229. #if HTS_XCONN
  2230.       if (back[i].status==STATUS_CONNECTING) {      // connexion
  2231.         do_wait=1;
  2232.  
  2233.         // noter socket write
  2234.         FD_SET(back[i].r.soc,&fds_c);
  2235.         
  2236.         // noter socket erreur
  2237.         FD_SET(back[i].r.soc,&fds_e);
  2238.  
  2239.         // calculer max
  2240.         if (max_c) {
  2241.           max_c=0;
  2242.           nfds=back[i].r.soc;
  2243.         } else if (back[i].r.soc>nfds) {
  2244.           // ID socket la plus ΘlevΘe
  2245.           nfds=back[i].r.soc;
  2246.         }
  2247.         
  2248.       } else
  2249. #endif
  2250. #if HTS_XGETHOST
  2251.       if (back[i].status==STATUS_WAIT_DNS) {      // attente
  2252.         // rien α faire..
  2253.       } else
  2254. #endif
  2255.       // poll pour la lecture sur les sockets
  2256.       if ((back[i].status>0) && (back[i].status<100)) {  // en rΘception http
  2257.             
  2258. #if BDEBUG==1
  2259.         //printf("....socket in progress: %d\n",back[i].r.soc);
  2260. #endif
  2261.         // non local et non ftp
  2262.         if (!back[i].r.is_file) {
  2263.         //## if (back[i].url_adr[0]!=lOCAL_CHAR) {
  2264.           
  2265.           // vΘrification de sΘcuritΘ
  2266.           if (back[i].r.soc!=INVALID_SOCKET) {  // hey, you never know..
  2267.             do_wait=1;
  2268.             
  2269.             // noter socket read
  2270.             FD_SET(back[i].r.soc,&fds);
  2271.             
  2272.             // noter socket error
  2273.             FD_SET(back[i].r.soc,&fds_e);
  2274.             
  2275.             // incrΘmenter nombre de sockets
  2276.             nsockets++;
  2277.  
  2278.             // calculer max
  2279.             if (max_c) {
  2280.               max_c=0;
  2281.               nfds=back[i].r.soc;
  2282.             } else if (back[i].r.soc>nfds) {
  2283.               // ID socket la plus ΘlevΘe
  2284.               nfds=back[i].r.soc;
  2285.             }
  2286.           } else {
  2287.             back[i].r.statuscode=STATUSCODE_CONNERROR;
  2288.             if (back[i].status==STATUS_CONNECTING)
  2289.               strcpybuff(back[i].r.msg,"Connect Error");
  2290.             else
  2291.               strcpybuff(back[i].r.msg,"Receive Error");
  2292.             back[i].status=STATUS_READY;  // terminΘ
  2293.             back_set_finished(sback, i);
  2294.             if ((opt->debug>0) && (opt->log!=NULL)) {
  2295.               HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"Unexpected socket error during pre-loop"LF); test_flush;
  2296.             }            
  2297.           }
  2298. #if WIDE_DEBUG
  2299.           else {
  2300.             DEBUG_W("PANIC!!! Socket is invalid in a poll test!\n");
  2301.           }
  2302. #endif
  2303.           
  2304.         }
  2305.         
  2306.       }
  2307.     }    
  2308.     nfds++;
  2309.     
  2310.     if (do_wait) {  // attendre
  2311.       // temps d'attente max: 2.5 seconde
  2312.       tv.tv_sec=HTS_SOCK_SEC;
  2313.       tv.tv_usec=HTS_SOCK_MS;
  2314.       
  2315. #if BDEBUG==1
  2316.       printf("..select\n");
  2317. #endif
  2318.       
  2319.       // poller les sockets-attention au noyau sous Unix..
  2320. #if HTS_WIDE_DEBUG    
  2321.       DEBUG_W("select\n");
  2322. #endif
  2323.       select(nfds,&fds,&fds_c,&fds_e,&tv);
  2324. #if HTS_WIDE_DEBUG    
  2325.       DEBUG_W("select done\n");
  2326. #endif      
  2327.     }
  2328.     
  2329.     // maximum data which can be received for a socket, if limited
  2330.     if (nsockets) {
  2331.       if (opt->maxrate>0) {
  2332.         max_read_bytes = ( check_downloadable_bytes(opt->maxrate) / nsockets );
  2333.         if (max_read_bytes > TAILLE_BUFFER) {
  2334.           /* limit size */
  2335.           max_read_bytes = TAILLE_BUFFER;
  2336.         } else if (max_read_bytes < TAILLE_BUFFER) {
  2337.           /* a small pause */
  2338.           Sleep(10);
  2339.         }
  2340.       }
  2341.     }
  2342.     if (!max_read_bytes)
  2343.       busy_recv=0;
  2344.     
  2345.     // recevoir les donnΘes arrivΘes
  2346.     for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  2347.     // for(i=0;i<back_max;i++) {
  2348.         unsigned int i = ( i_mod + mod_random ) % ( back_max );     
  2349.       if (back[i].status>0) {
  2350.         if (!back[i].r.is_file) {  // not file..
  2351.           if (back[i].r.soc!=INVALID_SOCKET) {  // hey, you never know..
  2352.             int err=FD_ISSET(back[i].r.soc,&fds_e);
  2353.             if (err) {
  2354.               if (back[i].r.soc!=INVALID_SOCKET) {
  2355. #if HTS_DEBUG_CLOSESOCK
  2356.                 DEBUG_W("back_wait: deletehttp\n");
  2357. #endif
  2358.                 deletehttp(&back[i].r);
  2359.               }
  2360.               back[i].r.soc=INVALID_SOCKET;
  2361.               back[i].r.statuscode=STATUSCODE_CONNERROR;
  2362.               if (back[i].status==STATUS_CONNECTING)
  2363.                 strcpybuff(back[i].r.msg,"Connect Error");
  2364.               else
  2365.                 strcpybuff(back[i].r.msg,"Receive Error");
  2366.               if (back[i].status == STATUS_ALIVE) {     /* Keep-alive socket */
  2367.                 back_delete(opt,cache,sback, i);
  2368.               } else {
  2369.                 back[i].status=STATUS_READY;  // terminΘ
  2370.                 back_set_finished(sback, i);
  2371.               }
  2372.             }
  2373.           }
  2374.         }
  2375.       }
  2376.       
  2377.       // ---- FLAG WRITE MIS A UN?: POUR LE CONNECT
  2378.       if (back[i].status==STATUS_CONNECTING) {   // attendre connect
  2379.         int dispo=0;
  2380.         // vΘrifier l'existance de timeout-check
  2381.         if (!gestion_timeout)
  2382.           if (back[i].timeout>0)
  2383.             gestion_timeout=1;
  2384.           
  2385.           // connectΘ?
  2386.           dispo=FD_ISSET(back[i].r.soc,&fds_c);
  2387.           if (dispo) {    // ok connected!!
  2388.             busy_state=1;
  2389.             
  2390. #if HTS_USEOPENSSL
  2391.             /* SSL mode */
  2392.             if (SSL_is_available && back[i].r.ssl) {
  2393.               // handshake not yet launched
  2394.               if (!back[i].r.ssl_con) {
  2395.                 SSL_CTX_set_options(openssl_ctx, SSL_OP_ALL);
  2396.                 // new session
  2397.                 back[i].r.ssl_con = SSL_new(openssl_ctx);
  2398.                 if (back[i].r.ssl_con) {
  2399.                   SSL_clear(back[i].r.ssl_con);
  2400.                   if (SSL_set_fd(back[i].r.ssl_con, back[i].r.soc) == 1) {
  2401.                     SSL_set_connect_state(back[i].r.ssl_con);
  2402.                     back[i].status = STATUS_SSL_WAIT_HANDSHAKE;         /* handshake wait */
  2403.                   } else
  2404.                     back[i].r.statuscode=STATUSCODE_SSL_HANDSHAKE;
  2405.                 } else
  2406.                   back[i].r.statuscode=STATUSCODE_SSL_HANDSHAKE;
  2407.               }
  2408.               /* Error */
  2409.               if (back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) {
  2410.                 strcpybuff(back[i].r.msg, "bad SSL/TLS handshake");
  2411.                 deletehttp(&back[i].r);
  2412.                 back[i].r.soc=INVALID_SOCKET;
  2413.                 back[i].r.statuscode=STATUSCODE_NON_FATAL;
  2414.                 back[i].status=STATUS_READY;
  2415.                 back_set_finished(sback, i);
  2416.               }
  2417.             }
  2418.             
  2419. #endif
  2420.  
  2421. #if BDEBUG==1
  2422.           printf("..connect ok on socket %d\n",back[i].r.soc);
  2423. #endif
  2424.           
  2425.           if ((back[i].r.soc != INVALID_SOCKET) && (back[i].status==STATUS_CONNECTING)) {
  2426.             /* limit nb. connections/seconds to avoid server overload */
  2427.             /*if (opt->maxconn>0) {
  2428.               Sleep(1000/opt->maxconn);
  2429.             }*/
  2430.             
  2431.             back[i].ka_time_start=time_local();
  2432.             if (back[i].timeout>0) {    // refresh timeout si besoin est
  2433.               back[i].timeout_refresh=back[i].ka_time_start;
  2434.             }
  2435.             if (back[i].rateout>0) {    // le taux de transfert de base sur le dΘbut de la connexion
  2436.               back[i].rateout_time=back[i].ka_time_start;
  2437.             }
  2438.             // envoyer header
  2439.             //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
  2440.             HTS_STAT.stat_nrequests++;
  2441.             if (!back[i].head_request)
  2442.               http_sendhead(opt, opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  2443.             else if (back[i].head_request==2)  // test en GET!
  2444.               http_sendhead(opt, opt->cookie,0,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  2445.             else        // test!
  2446.               http_sendhead(opt, opt->cookie,1,back[i].send_too,back[i].url_adr,back[i].url_fil,back[i].referer_adr,back[i].referer_fil,&back[i].r);         
  2447.             back[i].status=99;  // attendre en tΩte maintenant
  2448.           }
  2449.         }
  2450.         
  2451.         // attente gethostbyname
  2452.       }
  2453. #if HTS_USEOPENSSL
  2454.       else if (SSL_is_available && back[i].status == STATUS_SSL_WAIT_HANDSHAKE) {   // wait for SSL handshake
  2455.         /* SSL mode */
  2456.         if (back[i].r.ssl) {
  2457.           int conn_code;
  2458.           if ((conn_code = SSL_connect(back[i].r.ssl_con)) <= 0) {
  2459.             /* non blocking I/O, will retry */
  2460.             int err_code = SSL_get_error(back[i].r.ssl_con, conn_code);
  2461.             if (
  2462.               (err_code != SSL_ERROR_WANT_READ)
  2463.               &&
  2464.               (err_code != SSL_ERROR_WANT_WRITE)
  2465.               ) {
  2466.               char tmp[256];
  2467.               tmp[0]='\0';
  2468.               ERR_error_string(err_code, tmp);
  2469.               back[i].r.msg[0]='\0';
  2470.               strncatbuff(back[i].r.msg, tmp, sizeof(back[i].r.msg) - 2);
  2471.               if (!strnotempty(back[i].r.msg)) {
  2472.                 sprintf(back[i].r.msg, "SSL/TLS error %d", err_code);
  2473.               }
  2474.               deletehttp(&back[i].r);
  2475.               back[i].r.soc=INVALID_SOCKET;
  2476.               back[i].r.statuscode=STATUSCODE_NON_FATAL;
  2477.               back[i].status=STATUS_READY;
  2478.               back_set_finished(sback, i);
  2479.               }
  2480.           } else {        /* got it! */
  2481.             back[i].status=STATUS_CONNECTING;       // back to waitconnect
  2482.           }
  2483.         } else {
  2484.           strcpybuff(back[i].r.msg, "unexpected SSL/TLS error");
  2485.           deletehttp(&back[i].r);
  2486.           back[i].r.soc=INVALID_SOCKET;
  2487.           back[i].r.statuscode=STATUSCODE_NON_FATAL;
  2488.           back[i].status=STATUS_READY;
  2489.           back_set_finished(sback, i);
  2490.         }
  2491.         
  2492.       }
  2493. #endif
  2494. #if HTS_XGETHOST
  2495.       else if (back[i].status==STATUS_WAIT_DNS) {  // attendre gethostbyname
  2496. #if DEBUGDNS 
  2497.         //printf("status 101 for %s\n",back[i].url_adr);
  2498. #endif
  2499.  
  2500.         if (!gestion_timeout)
  2501.           if (back[i].timeout>0)
  2502.             gestion_timeout=1;
  2503.  
  2504.         if (host_wait(opt, &back[i])) {    // prΩt
  2505.           back[i].status=STATUS_CONNECTING;        // attente connexion
  2506.           if (back[i].timeout>0) {    // refresh timeout si besoin est
  2507.             back[i].timeout_refresh=time_local();
  2508.           }
  2509.           if (back[i].rateout>0) {    // le taux de transfert de base sur le dΘbut de la connexion
  2510.             back[i].rateout_time=time_local();
  2511.           }
  2512.  
  2513.           back[i].r.soc=http_xfopen(opt,0,0,0,back[i].send_too,back[i].url_adr,back[i].url_fil,&(back[i].r));
  2514.           if (back[i].r.soc==INVALID_SOCKET) {
  2515.             back[i].status=STATUS_READY;  // fini, erreur
  2516.             back_set_finished(sback, i);
  2517.             if (back[i].r.soc!=INVALID_SOCKET) {
  2518. #if HTS_DEBUG_CLOSESOCK
  2519.               DEBUG_W("back_wait(2): deletehttp\n");
  2520. #endif
  2521.               deletehttp(&back[i].r);
  2522.             }
  2523.             back[i].r.soc=INVALID_SOCKET;
  2524.             back[i].r.statuscode=STATUSCODE_NON_FATAL;
  2525.             if (strnotempty(back[i].r.msg)==0) 
  2526.               strcpybuff(back[i].r.msg,"Unable to resolve host name");
  2527.           }
  2528.         }
  2529.         
  2530.  
  2531.       // ---- FLAG READ MIS A UN?: POUR LA RECEPTION
  2532.       }
  2533. #endif
  2534. #if USE_BEGINTHREAD
  2535.       // ..rien α faire, c'est magic les threads
  2536. #else
  2537.       else if (back[i].status==STATUS_FTP_TRANSFER) {  // en rΘception ftp
  2538.         if (!fexist(back[i].location_buffer)) {    // terminΘ
  2539.           FILE* fp;
  2540.           fp=fopen(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"),"rb");
  2541.           if (fp) {
  2542.             int j=0;
  2543.             fscanf(fp,"%d ",&(back[i].r.statuscode));
  2544.             while(!feof(fp)) {
  2545.               int c = fgetc(fp);
  2546.               if (c!=EOF)
  2547.                 back[i].r.msg[j++]=c;
  2548.             }
  2549.             back[i].r.msg[j++]='\0';
  2550.             fclose(fp);
  2551.             unlink(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"));
  2552.             strcpybuff(fconcat(OPT_GET_BUFF(opt), back[i].location_buffer,".ok"),"");
  2553.           } else {
  2554.             strcpybuff(back[i].r.msg,"Unknown ftp result, check if file is ok");
  2555.             back[i].r.statuscode=STATUSCODE_INVALID;
  2556.           }
  2557.           back[i].status=STATUS_READY;
  2558.           back_set_finished(sback, i);
  2559.           // finalize transfer
  2560.           if (back[i].r.statuscode>0) {
  2561.             back_finalize(opt,cache,sback,i);
  2562.           }
  2563.         }
  2564.       }
  2565. #endif
  2566.       else if (back[i].status==STATUS_FTP_READY) {  // ftp ready
  2567.         back[i].status=STATUS_READY;
  2568.         back_set_finished(sback, i);
  2569.         // finalize transfer
  2570.         if (back[i].r.statuscode>0) {
  2571.           back_finalize(opt,cache,sback,i);
  2572.         }
  2573.       }
  2574.       else if ((back[i].status>0) && (back[i].status<1000)) {  // en rΘception http
  2575.         int dispo=0;
  2576.         
  2577.         // vΘrifier l'existance de timeout-check
  2578.         if (!gestion_timeout)
  2579.           if (back[i].timeout>0)
  2580.             gestion_timeout=1;
  2581.           
  2582.           // donnΘes dispo?
  2583.           //## if (back[i].url_adr[0]!=lOCAL_CHAR)
  2584.           if (back[i].r.is_file)
  2585.             dispo=1;
  2586.           else if (back[i].r.ssl)
  2587.             dispo=1;
  2588.           else
  2589.             dispo=FD_ISSET(back[i].r.soc,&fds);
  2590.  
  2591.           // Check transfer rate!
  2592.           if (!max_read_bytes)
  2593.             dispo=0;                // limit transfer rate
  2594.           
  2595.           if (dispo) {    // donnΘes dispo
  2596.             LLint retour_fread;
  2597.             busy_recv=1;    // on rΘcupΦre encore
  2598. #if BDEBUG==1
  2599.             printf("..data available on socket %d\n",back[i].r.soc);
  2600. #endif
  2601.  
  2602.             
  2603.             // range size hack old location
  2604.  
  2605. #if HTS_DIRECTDISK
  2606.             // Court-circuit:
  2607.             // Peut-on stocker le fichier directement sur disque?
  2608.             // Ahh que ca serait vachement mieux et que ahh que la mΘmoire vous dit merci!
  2609.             if (back[i].status) {
  2610.               if (back[i].r.is_write==0) {  // mode mΘmoire
  2611.                 if (back[i].r.adr==NULL) {  // rien n'a ΘtΘ Θcrit
  2612.                   if (!back[i].testmode) {  // pas mode test
  2613.                     if (strnotempty(back[i].url_sav)) {
  2614.                       if (strcmp(back[i].url_fil,"/robots.txt")) {
  2615.                         if (back[i].r.statuscode==HTTP_OK) {  // 'OK'
  2616.                           if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil)) {    // pas HTML
  2617.                             if (opt->getmode&2) {    // on peut ecrire des non html
  2618.                               int fcheck=0;
  2619.                               int last_errno = 0;
  2620.                               back[i].r.is_write=1;    // Θcrire
  2621.                               if (back[i].r.compressed
  2622.                                 &&
  2623.                                 /* .gz are *NOT* depacked!! */
  2624.                                 (strfield(get_ext(catbuff,back[i].url_sav),"gz") == 0)
  2625.                                 ) {
  2626.                                 back[i].tmpfile_buffer[0]='\0';
  2627.                                 back[i].tmpfile=tmpnam(back[i].tmpfile_buffer);
  2628.                                 if (back[i].tmpfile != NULL && back[i].tmpfile[0]) {
  2629.                                   if ((back[i].r.out=fopen(back[i].tmpfile,"wb")) == NULL) {
  2630.                                     last_errno = errno;
  2631.                                   }
  2632.                                 }
  2633.                               } else {
  2634.                                 file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 1, 1, back[i].r.notmodified);
  2635.                                 back[i].r.compressed=0;
  2636.                                 if ((back[i].r.out=filecreate(&opt->state.strc, back[i].url_sav)) == NULL) {
  2637.                                   last_errno = errno;
  2638.                                 }
  2639.                               }
  2640.                               if (back[i].r.out==NULL) {
  2641.                                 errno = last_errno;
  2642.                                 if ((fcheck=check_fatal_io_errno())) {
  2643.                                                                     HTS_LOG(opt,LOG_ERROR); fprintf(opt->log,"Mirror aborted: disk full or filesystem problems"LF); test_flush;
  2644.                                   opt->state.exit_xh=-1;   /* fatal error */
  2645.                                 }
  2646.                               }
  2647. #if HDEBUG
  2648.                               printf("direct-disk: %s\n",back[i].url_sav);
  2649. #endif
  2650.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  2651.                                 HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File received from net to disk: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2652.                               }
  2653.                               
  2654.                               if (back[i].r.out==NULL) {
  2655.                                 if (opt->log) {
  2656.                                   HTS_LOG(opt,LOG_ERROR);
  2657.                                   fprintf(opt->log,"Unable to save file %s : %s"LF,back[i].url_sav, strerror(last_errno));
  2658.                                   if (fcheck) {
  2659.                                     HTS_LOG(opt,LOG_ERROR);
  2660.                                     fprintf(opt->log,"* * Fatal write error, giving up"LF);
  2661.                                   }
  2662.                                   test_flush;
  2663.                                 }
  2664.                                 back[i].r.is_write=0;    // erreur, abandonner
  2665. #if HDEBUG
  2666.                                 printf("..error!\n");
  2667. #endif
  2668.                               }
  2669. #ifndef _WIN32
  2670.                               else chmod(back[i].url_sav,HTS_ACCESS_FILE);      
  2671. #endif          
  2672.                             } else {  // on coupe tout!
  2673.                               if ((opt->debug>1) && (opt->log!=NULL)) {
  2674.                                 HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File cancelled (non HTML): %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  2675.                               }
  2676.                               back[i].status=STATUS_READY;  // terminΘ
  2677.                               back_set_finished(sback, i);
  2678.                               if (!back[i].testmode)
  2679.                                 back[i].r.statuscode=STATUSCODE_INVALID;    // EUHH CANCEL
  2680.                               else
  2681.                                 back[i].r.statuscode=STATUSCODE_TEST_OK;    // "TEST OK"
  2682.                               if (back[i].r.soc!=INVALID_SOCKET) {
  2683. #if HTS_DEBUG_CLOSESOCK
  2684.                                 DEBUG_W("back_wait(3): deletehttp\n");
  2685. #endif
  2686.                                 deletehttp(&back[i].r);
  2687.                               }
  2688.                               back[i].r.soc=INVALID_SOCKET;
  2689.                             }
  2690.                           }
  2691.                         }
  2692.                       }
  2693.                     }
  2694.                   }
  2695.                 }
  2696.               }
  2697.             }
  2698. #endif              
  2699.  
  2700.             // rΘception de donnΘes depuis socket ou fichier
  2701.             if (back[i].status) {
  2702.               if (back[i].status==STATUS_WAIT_HEADERS)  // recevoir par bloc de lignes
  2703.                 retour_fread=http_xfread1(&(back[i].r),0);
  2704.               else if (back[i].status==STATUS_CHUNK_WAIT || back[i].status==STATUS_CHUNK_CR) { // recevoir longueur chunk en hexa caractΦre par caractΦre
  2705.                 // backuper pour lire dans le buffer chunk
  2706.                 htsblk r;
  2707.                 memcpy(&r, &(back[i].r), sizeof(htsblk));
  2708.                 back[i].r.is_write=0;                   // mΘmoire
  2709.                 back[i].r.adr=back[i].chunk_adr;        // adresse
  2710.                 back[i].r.size=back[i].chunk_size;      // taille taille chunk
  2711.                 back[i].r.totalsize=-1;                 // total inconnu
  2712.                 back[i].r.out=NULL;
  2713.                 back[i].r.is_file=0;
  2714.                 //
  2715.                 // ligne par ligne
  2716.                 retour_fread=http_xfread1(&(back[i].r),-1);
  2717.                 // modifier et restaurer
  2718.                 back[i].chunk_adr=back[i].r.adr;        // adresse
  2719.                 back[i].chunk_size=back[i].r.size;      // taille taille chunk
  2720.                 memcpy(&(back[i].r), &r, sizeof(htsblk));    // restaurer vΘritable r
  2721.               }
  2722.               else if (back[i].is_chunk) {         // attention chunk, limiter taille α lire
  2723. #if CHUNKDEBUG==1
  2724.                 printf("[%d] read %d bytes\n",(int)back[i].r.soc,(int)min(back[i].r.totalsize-back[i].r.size,max_read_bytes));
  2725. #endif
  2726.                 retour_fread=(int) http_xfread1(&(back[i].r),(int) min(back[i].r.totalsize-back[i].r.size,max_read_bytes));
  2727.               } else              
  2728.                 retour_fread=(int) http_xfread1(&(back[i].r),(int) max_read_bytes);
  2729.                 // retour_fread=http_fread1(&(back[i].r));
  2730.             } else
  2731.               retour_fread=READ_EOF;                    // interruption ou annulation interne (peut ne pas Ωtre une erreur)
  2732.             
  2733.             // Si rΘception chunk, tester si on est pas α la fin!
  2734.             if (back[i].status==1) {
  2735.               if (back[i].is_chunk) {     // attendre prochain chunk
  2736.                 if (back[i].r.size==back[i].r.totalsize) {      // fin chunk!
  2737.                   //printf("chunk end at %d\n",back[i].r.size);
  2738.                   back[i].status=STATUS_CHUNK_CR;  /* fetch ending CRLF */
  2739.                   if (back[i].chunk_adr!=NULL) { 
  2740.                     freet(back[i].chunk_adr); 
  2741.                     back[i].chunk_adr=NULL; 
  2742.                   } 
  2743.                   back[i].chunk_size=0;
  2744.                   retour_fread=0;       // pas d'erreur
  2745. #if CHUNKDEBUG==1
  2746.                   printf("[%d] waiting for current chunk CRLF..\n",(int)back[i].r.soc);
  2747. #endif
  2748.                 }
  2749.               } else if (back[i].r.keep_alive) {
  2750.                 if (back[i].r.size==back[i].r.totalsize) {      // fin!
  2751.                   retour_fread=READ_EOF;       // end
  2752.                 }
  2753.               }
  2754.             }
  2755.             
  2756.             if (retour_fread < 0) {    // fin rΘception
  2757.               back[i].status=STATUS_READY;    // terminΘ
  2758.               back_set_finished(sback, i);
  2759.              /*KA back[i].r.soc=INVALID_SOCKET; */
  2760. #if CHUNKDEBUG==1
  2761.               if (back[i].is_chunk)
  2762.                 printf("[%d] must be the last chunk for %s (connection closed) - %d/%d\n",(int)back[i].r.soc,back[i].url_fil,back[i].r.size,back[i].r.totalsize);
  2763. #endif
  2764.               if (retour_fread < 0 && retour_fread != READ_EOF) {
  2765.                 if (back[i].r.size > 0)
  2766.                   strcpybuff(back[i].r.msg, "Interrupted transfer");
  2767.                 else
  2768.                   strcpybuff(back[i].r.msg, "No data (connection closed)");
  2769.                 back[i].r.statuscode=STATUSCODE_CONNERROR;
  2770.               } else if ((back[i].r.statuscode <= 0) && (strnotempty(back[i].r.msg)==0)) {
  2771. #if HDEBUG
  2772.                 printf("error interruped: %s\n",back[i].r.adr);
  2773. #endif        
  2774.                 if (back[i].r.size>0)
  2775.                   strcpybuff(back[i].r.msg,"Interrupted transfer");
  2776.                 else
  2777.                   strcpybuff(back[i].r.msg,"No data (connection closed)");
  2778.                 back[i].r.statuscode=STATUSCODE_CONNERROR;
  2779.               }
  2780.  
  2781.               // Close socket
  2782.               if (back[i].r.soc!=INVALID_SOCKET) {
  2783. #if HTS_DEBUG_CLOSESOCK
  2784.                 DEBUG_W("back_wait(4): deletehttp\n");
  2785. #endif
  2786.                 /*KA deletehttp(&back[i].r);*/
  2787.                 back_maydeletehttp(opt, cache, sback, i);
  2788.               }
  2789.  
  2790.               // finalize transfer
  2791.               if (back[i].r.statuscode>0
  2792.                 && !IS_DELAYED_EXT(back[i].url_sav)
  2793.                 ) {
  2794.                 back_finalize(opt,cache,sback,i);
  2795.               }
  2796.  
  2797.               if (back[i].r.totalsize>0) {    // tester totalsize
  2798.               //if ((back[i].r.totalsize>0) && (back[i].status==STATUS_WAIT_HEADERS)) {    // tester totalsize
  2799.                 if (back[i].r.totalsize!=back[i].r.size) {  // pas la mΩme!
  2800.                   if (!opt->tolerant) {
  2801.                     //#if HTS_CL_IS_FATAL
  2802.                     deleteaddr(&back[i].r);
  2803.                     if (back[i].r.size<back[i].r.totalsize)
  2804.                       back[i].r.statuscode=STATUSCODE_CONNERROR;        // recatch
  2805.                     sprintf(back[i].r.msg,"Incorrect length ("LLintP" Bytes, "LLintP" expected)",(LLint)back[i].r.size,(LLint)back[i].r.totalsize);
  2806.                   } else {
  2807.                     //#else
  2808.                     // Un warning suffira..
  2809.                     if (cache->log!=NULL) {
  2810.                       fspc(opt,cache->log,"warning"); fprintf(cache->log,"Incorrect length ("LLintP"!="LLintP" expected) for %s%s"LF,(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2811.                     }
  2812.                     //#endif
  2813.                   }
  2814.                 }
  2815.               }
  2816. #if BDEBUG==1
  2817.               printf("transfer ok\n");
  2818. #endif
  2819.             } else if (retour_fread > 0) {    // pas d'erreur de rΘception et data
  2820.               if (back[i].timeout>0) {    // refresh timeout si besoin est
  2821.                 back[i].timeout_refresh=time_local();
  2822.               }
  2823.  
  2824.               // Traitement des en tΩtes chunks ou en tΩtes
  2825.               if (back[i].status==STATUS_CHUNK_WAIT || back[i].status==STATUS_CHUNK_CR) {        // rΘception taille chunk en hexa (  aprΦs les en tΩtes, peut ne pas
  2826.                 if (back[i].chunk_size > 0 && back[i].chunk_adr[back[i].chunk_size-1]==10) {
  2827.                   int chunk_size=-1;
  2828.                   char chunk_data[64];
  2829.                   if (back[i].chunk_size<32) {      // pas trop gros
  2830.                     char* chstrip=back[i].chunk_adr;
  2831.                     back[i].chunk_adr[ back[i].chunk_size-1]='\0';    // octet nul 
  2832.                     // skip leading spaces or cr
  2833.                     while(isspace(*chstrip)) chstrip++;
  2834.                     chunk_data[0] = '\0';
  2835.                     strncatbuff(chunk_data, chstrip, sizeof(chunk_data) - 2);
  2836.                     // strip chunk-extension
  2837.                     while( (chstrip = strchr(chunk_data, ';'))) *chstrip='\0';
  2838.                     while( (chstrip = strchr(chunk_data, ' '))) *chstrip='\0';
  2839.                     while( (chstrip = strchr(chunk_data, '\r'))) *chstrip='\0';
  2840. #if CHUNKDEBUG==1
  2841.                     printf("[%d] chunk received and read: %s\n",(int)back[i].r.soc,chunk_data);
  2842. #endif
  2843.                     if (back[i].r.totalsize<0)
  2844.                       back[i].r.totalsize=0;        // initialiser α 0
  2845.                     if (back[i].status==STATUS_CHUNK_WAIT) {   // "real" chunk
  2846.                       if (sscanf(chunk_data,"%x",&chunk_size) == 1) {
  2847.                         if (chunk_size > 0)
  2848.                           back[i].chunk_blocksize = chunk_size;  /* the data block chunk size */
  2849.                                                 else
  2850.                                                     back[i].chunk_blocksize = -1;  /* ending */
  2851.                                                 back[i].r.totalsize+=chunk_size;    // noter taille
  2852.                                                 if (back[i].r.adr != NULL || !back[i].r.is_write) {  // Not to disk
  2853.                                                     back[i].r.adr=(char*) realloct(back[i].r.adr, (size_t)back[i].r.totalsize + 1);
  2854.                                                     if (!back[i].r.adr) {
  2855.                                                         if (cache->log!=NULL) {
  2856.                                                             fprintf(cache->log,"Error: Not enough memory ("LLintP") for %s%s"LF,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2857.                                                         }
  2858.                                                     }
  2859.                                                 }
  2860. #if CHUNKDEBUG==1
  2861.                         printf("[%d] chunk length: %d - next total "LLintP":\n",(int)back[i].r.soc,(int)chunk_size,(LLint)back[i].r.totalsize);
  2862. #endif
  2863.                       } else {
  2864.                         if (cache->log!=NULL) {
  2865.                           fprintf(cache->log,"Warning: Illegal chunk (%s) for %s%s"LF,back[i].chunk_adr,back[i].url_adr,back[i].url_fil);
  2866.                         }
  2867.                       }
  2868.                     } else {   /* back[i].status==STATUS_CHUNK_CR : just receiving ending CRLF after data */
  2869.                       if (chunk_data[0] == '\0') {
  2870.                         if (back[i].chunk_blocksize > 0)
  2871.                           chunk_size=(int)back[i].chunk_blocksize;  /* recent data chunk size */
  2872.                         else if (back[i].chunk_blocksize == -1)
  2873.                           chunk_size=0;                        /* ending chunk */
  2874.                         else
  2875.                           chunk_size=1;                        /* fake positive size for 1st chunk history */
  2876. #if CHUNKDEBUG==1
  2877.                         printf("[%d] chunk CRLF seen\n", (int)back[i].r.soc);
  2878. #endif
  2879.                       } else {
  2880.                         if (cache->log!=NULL) {
  2881.                           fprintf(cache->log,"Warning: Illegal chunk CRLF (%s) for %s%s"LF,back[i].chunk_adr,back[i].url_adr,back[i].url_fil);
  2882.                         }
  2883. #if CHUNKDEBUG==1
  2884.                         printf("[%d] chunk CRLF ERROR!! : '%s'\n", (int)back[i].r.soc, chunk_data);
  2885. #endif
  2886.                       }
  2887.                     }
  2888.                   } else {                                  
  2889.                     if (cache->log!=NULL) {
  2890.                       fprintf(cache->log,"Warning: Chunk too big ("LLintP") for %s%s"LF,(LLint)back[i].chunk_size,back[i].url_adr,back[i].url_fil);
  2891.                     }
  2892.                   }
  2893.                     
  2894.                   // ok, continuer sur le body
  2895.                     
  2896.                   // si chunk non nul continuer (ou commencer)
  2897.                   if (back[i].status==STATUS_CHUNK_CR && chunk_size > 0) {
  2898.                     back[i].status = STATUS_CHUNK_WAIT;  /* waiting for next chunk (NN\r\n<data>\r\nNN\r\n<data>..\r\n0\r\n\r\n) */
  2899. #if CHUNKDEBUG==1
  2900.                     printf("[%d] waiting for next chunk\n", (int)back[i].r.soc);
  2901. #endif
  2902.                   } else if (back[i].status==STATUS_CHUNK_WAIT && chunk_size == 0) {  /* final chunk */
  2903.                     back[i].status=STATUS_CHUNK_CR;  /* final CRLF */
  2904. #if CHUNKDEBUG==1
  2905.                     printf("[%d] waiting for final CRLF (chunk)\n", (int)back[i].r.soc);
  2906. #endif
  2907.                   } else if (back[i].status==STATUS_CHUNK_WAIT && chunk_size >= 0) {  /* will fetch data now */
  2908.                     back[i].status=1;     // continuer body    
  2909. #if CHUNKDEBUG==1
  2910.                     printf("[%d] waiting for body (chunk)\n", (int)back[i].r.soc);
  2911. #endif
  2912.                   } else {                /* zero-size-chunk-CRLF (end) or error */
  2913. #if CHUNKDEBUG==1
  2914.                     printf("[%d] chunk end, total: %d\n",(int)back[i].r.soc,back[i].r.size);
  2915. #endif
  2916.                     /* End */
  2917.                     //if (back[i].status==STATUS_CHUNK_CR) {
  2918.                     back[i].status=STATUS_READY;     // fin  
  2919.                     back_set_finished(sback, i);
  2920.                     //}
  2921.  
  2922.                     // finalize transfer if not temporary
  2923.                     if (!IS_DELAYED_EXT(back[i].url_sav)) {
  2924.                       back_finalize(opt,cache,sback,i);
  2925.                     } else {
  2926.                       if (back[i].r.statuscode == HTTP_OK) {
  2927.                         if (cache->log!=NULL) {
  2928.                           fspc(opt,cache->log,"warning"); fprintf(cache->log,"Unexpected incomplete type with 200 code at %s%s"LF, back[i].url_adr, back[i].url_fil);
  2929.                         }
  2930.                       }
  2931.                     }
  2932.                     if (back[i].r.soc!=INVALID_SOCKET) {
  2933. #if HTS_DEBUG_CLOSESOCK
  2934.                       DEBUG_W("back_wait(5): deletehttp\n");
  2935. #endif
  2936.                       /* Error */
  2937.                       if (chunk_size < 0) {
  2938.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  2939.                         deleteaddr(&back[i].r);
  2940.                         back[i].r.statuscode=STATUSCODE_INVALID;
  2941.                         strcpybuff(back[i].r.msg,"Invalid chunk");
  2942. #if CHUNKDEBUG==1
  2943.                         printf("[%d] chunk error\n", (int)back[i].r.soc);
  2944. #endif
  2945.                       } else /* if chunk_size == 0 */ {
  2946. #if CHUNKDEBUG==1
  2947.                         printf("[%d] all chunks now received\n", (int)back[i].r.soc);
  2948. #endif
  2949.                           
  2950.                         /* Tester totalsize en fin de chunk */
  2951.                         if ((back[i].r.totalsize>0)) {    // tester totalsize
  2952.                           if (back[i].r.totalsize!=back[i].r.size) {  // pas la mΩme!
  2953.                             if (!opt->tolerant) {
  2954.                               deleteaddr(&back[i].r);
  2955.                               back[i].r.statuscode=STATUSCODE_INVALID;
  2956.                               strcpybuff(back[i].r.msg,"Incorrect length");
  2957.                             } else {
  2958.                               // Un warning suffira..
  2959.                               if (cache->log!=NULL) {
  2960.                                 fspc(opt,cache->log,"warning"); fprintf(cache->log,"Incorrect length ("LLintP"!="LLintP" expected) for %s%s"LF,(LLint)back[i].r.size,(LLint)back[i].r.totalsize,back[i].url_adr,back[i].url_fil);
  2961.                               }
  2962.                             }
  2963.                           }
  2964.                         }
  2965.                           
  2966.                         /* Oops, trailers! */
  2967.                         if (back[i].r.keep_alive_trailers) {
  2968.                           /* fixme (not yet supported) */
  2969.                         }
  2970.                           
  2971.                       }
  2972.                         
  2973.                         
  2974.                     }
  2975.                   }
  2976.                     
  2977.                   // effacer buffer (chunk en tete)
  2978.                   if (back[i].chunk_adr!=NULL) {
  2979.                     freet(back[i].chunk_adr);
  2980.                     back[i].chunk_adr=NULL;
  2981.                     back[i].chunk_size=0;
  2982.                     // NO! xxback[i].chunk_blocksize = 0;
  2983.                   }
  2984.                   
  2985.                 } // taille buffer chunk > 1 && LF
  2986.                 //
  2987.               } else if (back[i].status==STATUS_WAIT_HEADERS) {        // en tΩtes (avant le chunk si il est prΘsent)
  2988.                 //
  2989.                 if (back[i].r.size>=2) {
  2990.                   // double LF
  2991.                   if (
  2992.                     ((back[i].r.adr[back[i].r.size-1]==10) && (back[i].r.adr[back[i].r.size-2]==10)) 
  2993.                     ||
  2994.                     (back[i].r.adr[0] == '<')    /* bogus server */
  2995.                     ) {
  2996.                     char rcvd[2048];
  2997.                     int ptr=0;
  2998.                     int noFreebuff=0;
  2999.                     
  3000. #if BDEBUG==1
  3001.                     printf("..ok, header received\n");
  3002. #endif
  3003.                     
  3004.                     
  3005.                     /* Hack for zero-length headers */
  3006.                     if (back[i].status != 0 && back[i].r.adr[0] != '<') {
  3007.                       
  3008.                       // ----------------------------------------
  3009.                       // traiter en-tΩte!
  3010.                       // status-line α rΘcupΘrer
  3011.                       ptr+=binput(back[i].r.adr+ptr,rcvd,2000);
  3012.                       if (strnotempty(rcvd)==0) {
  3013.                         /* Bogus CRLF, OR recycled connection and trailing chunk CRLF */
  3014.                         ptr+=binput(back[i].r.adr+ptr,rcvd,2000);
  3015.                       }
  3016.                       
  3017.                       // traiter status-line
  3018.                       treatfirstline(&back[i].r,rcvd);
  3019.                       
  3020. #if HDEBUG
  3021.                       printf("(Buffer) Status-Code=%d\n",back[i].r.statuscode);
  3022. #endif
  3023.                       if (_DEBUG_HEAD) {
  3024.                         if (ioinfo) {
  3025.                           fprintf(ioinfo,"[%d] response for %s%s:\r\ncode=%d\r\n",
  3026.                             back[i].r.debugid, jump_identification(back[i].url_adr),back[i].url_fil,back[i].r.statuscode);
  3027.                           fprintfio(ioinfo,back[i].r.adr,">>> ");
  3028.                           fprintf(ioinfo,"\r\n");
  3029.                           fflush(ioinfo);
  3030.                         }                    // en-tΩte
  3031.                       }
  3032.                       
  3033.                       // header // ** !attention! HTTP/0.9 non supportΘ
  3034.                       do {
  3035.                         ptr+=binput(back[i].r.adr+ptr,rcvd,2000);          
  3036. #if HDEBUG
  3037.                         printf("(buffer)>%s\n",rcvd);      
  3038. #endif
  3039.                         /*
  3040.                         if (_DEBUG_HEAD) {
  3041.                         if (ioinfo) {
  3042.                         fprintf(ioinfo,"(buffer)>%s\r\n",rcvd);      
  3043.                         fflush(ioinfo);
  3044.                         }
  3045.                         }
  3046.                         */
  3047.                         
  3048.                         if (strnotempty(rcvd))
  3049.                           treathead(opt->cookie,back[i].url_adr,back[i].url_fil,&back[i].r,rcvd);  // traiter
  3050.                         
  3051.                         // parfois les serveurs buggΘs renvoient un content-range avec un 200
  3052.                         if (back[i].r.statuscode==HTTP_OK)  // 'OK'
  3053.                           if (strfield(rcvd,"content-range:"))  // Avec un content-range: relisez les RFC..
  3054.                             back[i].r.statuscode=206;    // FORCER A 206 !!!!!
  3055.                           
  3056.                       } while(strnotempty(rcvd));
  3057.                       // ----------------------------------------                    
  3058.  
  3059.                     } else {
  3060.                       // assume text/html, OK
  3061.                       treatfirstline(&back[i].r, back[i].r.adr);
  3062.                       noFreebuff=1;
  3063.                     }
  3064.                       
  3065.                     // Callback
  3066.                     {
  3067.                       int test_head = RUN_CALLBACK6(opt, receivehead, 
  3068.                         back[i].r.adr, back[i].url_adr, back[i].url_fil, back[i].referer_adr, back[i].referer_fil, &back[i].r);
  3069.                       if (test_head!=1) {
  3070.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  3071.                           HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"External wrapper aborted transfer, breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3072.                         }
  3073.                         back[i].status=STATUS_READY;  // FINI
  3074.                         back_set_finished(sback, i);
  3075.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3076.                         strcpybuff(back[i].r.msg,"External wrapper aborted transfer");
  3077.                         back[i].r.statuscode = STATUSCODE_INVALID;
  3078.                       }
  3079.                     }
  3080.  
  3081.                     // Free headers memory now
  3082.                     // Actually, save them for informational purpose
  3083.                     if (!noFreebuff) {
  3084.                       char* block = back[i].r.adr;
  3085.                       back[i].r.adr = NULL;
  3086.                       deleteaddr(&back[i].r);
  3087.                       back[i].r.headers = block;
  3088.                     }                  
  3089.                     
  3090.                     /* 
  3091.                     Status code and header-response hacks
  3092.                     */
  3093.  
  3094.                     
  3095.                     // Check response : 203 == 200
  3096.                     if (back[i].r.statuscode==HTTP_NON_AUTHORITATIVE_INFORMATION) {
  3097.                       back[i].r.statuscode=HTTP_OK;       // forcer "OK"
  3098.                     } else if (back[i].r.statuscode == HTTP_CONTINUE) {
  3099.                       back[i].status=STATUS_WAIT_HEADERS;
  3100.                       back[i].r.size=0;
  3101.                       back[i].r.totalsize=0;
  3102.                       back[i].chunk_size=0;
  3103.                       back[i].r.statuscode=STATUSCODE_INVALID;
  3104.                       back[i].r.msg[0]='\0';
  3105.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  3106.                         HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"Status 100 detected for %s%s, continuing headers"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3107.                       }
  3108.                       continue;
  3109.                     }
  3110.                     
  3111.                     /*
  3112.                     Solve "false" 416 problems
  3113.                     */
  3114.                     if (back[i].r.statuscode==416) {  // 'Requested Range Not Satisfiable'
  3115.                       // Example:
  3116.                       // Range: bytes=2830-
  3117.                       // ->
  3118.                       // Content-Range: bytes */2830
  3119.                       if (back[i].range_req_size == back[i].r.crange) {
  3120.                         filenote(&opt->state.strc,back[i].url_sav,NULL);
  3121.                         file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);
  3122.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3123.                         back[i].status=STATUS_READY;    // READY
  3124.                         back_set_finished(sback, i);
  3125.                         back[i].r.size=back[i].r.totalsize=back[i].range_req_size;
  3126.                         back[i].r.statuscode=HTTP_NOT_MODIFIED;     // NOT MODIFIED
  3127.                         if ((opt->debug>1) && (opt->log!=NULL)) {
  3128.                           HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File seems complete (good 416 message), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3129.                         }
  3130.                       }
  3131.                     }
  3132.                     
  3133.                     // transform 406 into 200 ; we'll catch embedded links inside the choice page
  3134.                     if (back[i].r.statuscode==406) {  // 'Not Acceptable'
  3135.                       back[i].r.statuscode=HTTP_OK;
  3136.                     }
  3137.  
  3138.                                         // 'do not erase already downloaded file'
  3139.                                         // on an updated file
  3140.                                         // with an error : consider a 304 error
  3141.                                         if (!opt->delete_old) {
  3142.                                             if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update && !back[i].testmode) {
  3143.                                                 if (back[i].url_sav[0] && fexist(back[i].url_sav)) {
  3144.                                                     if ((opt->debug>1) && (opt->log!=NULL)) {
  3145.                                                         HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"Error ignored %d (%s) because of 'no purge' option for %s%s"LF,back[i].r.statuscode,back[i].r.msg,back[i].url_adr,back[i].url_fil); test_flush;
  3146.                                                     }
  3147.                                                     back[i].r.statuscode = HTTP_NOT_MODIFIED;
  3148.                                                     deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3149.                                                 }
  3150.                                             }
  3151.                                         }
  3152.  
  3153.                                         // Various hacks to limit re-transfers when updating a mirror
  3154.                     // Force update if same size detected
  3155.                     if (opt->sizehack) {
  3156.                       // We already have the file
  3157.                       // and ask the remote server for an update
  3158.                       // Some servers, especially dynamic pages severs, always
  3159.                       // answer that the page has been modified since last visit
  3160.                       // And answer with a 200 (OK) response, and the same page
  3161.                       // If the size is the same, and the option has been set, we assume
  3162.                       // that the file is identical - and therefore let's break the connection
  3163.                       if (back[i].is_update) {          // mise α jour
  3164.                         if (back[i].r.statuscode==HTTP_OK && !back[i].testmode) {  // 'OK'
  3165.                           htsblk r = cache_read(opt,cache,back[i].url_adr,back[i].url_fil,NULL,NULL);    // lire entrΘe cache
  3166.                           if (r.statuscode == HTTP_OK) {  // OK pas d'erreur cache
  3167.                             LLint len1,len2;
  3168.                             len1=r.totalsize;
  3169.                             len2=back[i].r.totalsize;
  3170.                             if (r.size>0)
  3171.                               len1=r.size;
  3172.                             if (len1>0) {
  3173.                               if (len1 == len2) {             // tailles identiques
  3174.                                 back[i].r.statuscode=HTTP_NOT_MODIFIED;     // forcer NOT MODIFIED
  3175.                                 deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3176.                                 if ((opt->debug>1) && (opt->log!=NULL)) {
  3177.                                   HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File seems complete (same size), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3178.                                 }
  3179.                               }
  3180.                             }
  3181.                           } else {
  3182.                             if (opt->log!=NULL) {
  3183.                               HTS_LOG(opt,LOG_WARNING); fprintf(opt->log,"File seems complete (same size), but there was a cache read error: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3184.                             }
  3185.                           }
  3186.                           if (r.adr) {
  3187.                             freet(r.adr);
  3188.                                                         r.adr = NULL;
  3189.                           }
  3190.                         }
  3191.                       }
  3192.                     }
  3193.                     
  3194.                     // Various hacks to limit re-transfers when updating a mirror
  3195.                     // Detect already downloaded file (with another browser, for example)
  3196.                     if (opt->sizehack) {
  3197.                       if (!back[i].is_update) {          // mise α jour
  3198.                         if (back[i].r.statuscode==HTTP_OK && !back[i].testmode) {  // 'OK'
  3199.                           if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil)) {    // not HTML
  3200.                             if (strnotempty(back[i].url_sav)) {  // target found
  3201.                               int size = fsize(back[i].url_sav);  // target size
  3202.                               if (size >= 0) {
  3203.                                 if (back[i].r.totalsize == size) {  // same size!
  3204.                                   deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3205.                                   back[i].status=STATUS_READY;    // READY
  3206.                                   back_set_finished(sback, i);
  3207.                                   back[i].r.size=back[i].r.totalsize;
  3208.                                   filenote(&opt->state.strc,back[i].url_sav,NULL);
  3209.                                   file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);
  3210.                                   back[i].r.statuscode=HTTP_NOT_MODIFIED;     // NOT MODIFIED
  3211.                                   if ((opt->debug>1) && (opt->log!=NULL)) {
  3212.                                     HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File seems complete (same size file discovered), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3213.                                   }
  3214.                                 }
  3215.                               }
  3216.                             }
  3217.                           }
  3218.                         }
  3219.                       }
  3220.                     }
  3221.                     
  3222.                     // Various hacks to limit re-transfers when updating a mirror
  3223.                     // Detect bad range: header
  3224.                     if (opt->sizehack) {
  3225.                       // We have request for a partial file (with a 'Range: NNN-' header)
  3226.                       // and received a complete file notification (200), with 'Content-length: NNN'
  3227.                       // it might be possible that we had the complete file
  3228.                       // this is the case in *most* cases, so break the connection
  3229.                       if (back[i].r.is_write==0) {  // mode mΘmoire
  3230.                         if (back[i].r.adr==NULL) {  // rien n'a ΘtΘ Θcrit
  3231.                           if (!back[i].testmode) {  // pas mode test
  3232.                             if (strnotempty(back[i].url_sav)) {
  3233.                               if (strcmp(back[i].url_fil,"/robots.txt")) {
  3234.                                 if (back[i].r.statuscode==HTTP_OK) {  // 'OK'
  3235.                                   if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_fil)) {    // pas HTML
  3236.                                     if (back[i].r.statuscode==HTTP_OK) {      // "OK"
  3237.                                       if (back[i].range_req_size>0) {     // but Range: requested
  3238.                                         if (back[i].range_req_size == back[i].r.totalsize) {    // And same size
  3239. #if HTS_DEBUG_CLOSESOCK
  3240.                                           DEBUG_W("back_wait(skip_range): deletehttp\n");
  3241. #endif
  3242.                                           deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3243.                                           back[i].status=STATUS_READY;    // READY
  3244.                                           back_set_finished(sback, i);
  3245.                                           back[i].r.size=back[i].r.totalsize;
  3246.                                           filenote(&opt->state.strc,back[i].url_sav,NULL);
  3247.                                           file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);
  3248.                                           back[i].r.statuscode=HTTP_NOT_MODIFIED;     // NOT MODIFIED
  3249.                                           if ((opt->debug>1) && (opt->log!=NULL)) {
  3250.                                             HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File seems complete (reget failed), breaking connection: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3251.                                           }
  3252.                                         }
  3253.                                       }
  3254.                                     }
  3255.                                     
  3256.                                   }
  3257.                                 }
  3258.                               }
  3259.                             }
  3260.                           }
  3261.                         }
  3262.                       }
  3263.                     }
  3264.                     // END - Various hacks to limit re-transfers when updating a mirror
  3265.  
  3266.                     /* 
  3267.                     End of status code and header-response hacks
  3268.                     */
  3269.  
  3270.                     
  3271.                     
  3272.                     /* Interdiction taille par le wizard? */
  3273.                     if (back[i].r.soc!=INVALID_SOCKET) {
  3274.                       if (!back_checksize(opt,&back[i],1)) {
  3275.                         back[i].status=STATUS_READY;  // FINI
  3276.                         back_set_finished(sback, i);
  3277.                         back[i].r.statuscode=STATUSCODE_TOO_BIG;
  3278.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3279.                         if (!back[i].testmode)
  3280.                           strcpybuff(back[i].r.msg,"File too big");
  3281.                         else
  3282.                           strcpybuff(back[i].r.msg,"Test: File too big");
  3283.                       }
  3284.                     }
  3285.                     
  3286.                     /* sinon, continuer */
  3287.                     /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok rΘcupΘrer body? */
  3288.                     // head: terminΘ
  3289.                     if (back[i].head_request) {
  3290.                       if ((opt->debug>1) && (opt->log!=NULL)) {
  3291.                         HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"Tested file: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3292.                       }
  3293. #if HTS_DEBUG_CLOSESOCK
  3294.                       DEBUG_W("back_wait(head request): deletehttp\n");
  3295. #endif
  3296.                       // Couper connexion
  3297.                       if (!back[i].http11) {    /* NO KA */
  3298.                         deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3299.                       }
  3300.                       back[i].status=STATUS_READY;  // terminΘ
  3301.                       back_set_finished(sback, i);
  3302.                     }
  3303.                     // traiter une Θventuelle erreur 304 (cache α jour utilisable)
  3304.                     else if (back[i].r.statuscode==HTTP_NOT_MODIFIED) {  // document α jour dans le cache
  3305.                       // lire dans le cache
  3306.                       // ** NOTE: pas de vΘrif de la taille ici!!
  3307. #if HTS_DEBUG_CLOSESOCK
  3308.                       DEBUG_W("back_wait(file is not modified): deletehttp\n");
  3309. #endif
  3310.                       /* clear everything but connection: switch, close, and reswitch */
  3311.                       {
  3312.                         htsblk tmp;
  3313.                         memset(&tmp, 0, sizeof(tmp));
  3314.                         back_connxfr(&back[i].r, &tmp);
  3315.                         back[i].r=cache_read(opt,cache,back[i].url_adr,back[i].url_fil,back[i].url_sav,back[i].location_buffer);
  3316.                         back[i].r.location=back[i].location_buffer;
  3317.                         back_connxfr(&tmp,&back[i].r);
  3318.                       }
  3319.  
  3320.                       // hack:
  3321.                       // In case of 'if-unmodified-since' hack, a 304 status can be sent
  3322.                       // then, force 'ok' status
  3323.                       if (back[i].r.statuscode == STATUSCODE_INVALID) {
  3324.                         if (fexist(back[i].url_sav)) {
  3325.                           back[i].r.statuscode=HTTP_OK;     // OK
  3326.                           strcpybuff(back[i].r.msg, "OK (cached)");
  3327.                           back[i].r.is_file=1;
  3328.                           back[i].r.totalsize = back[i].r.size = fsize(back[i].url_sav);
  3329.                           get_httptype(opt,back[i].r.contenttype, back[i].url_sav, 1);
  3330.                           if ((opt->debug>0) && (opt->log!=NULL)) {
  3331.                             HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"Not-modified status without cache guessed: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3332.                           }
  3333.                         }
  3334.                       }
  3335.  
  3336.                       // Status is okay?
  3337.                       if (back[i].r.statuscode!=-1) { // pas d'erreur de lecture
  3338.                         back[i].status=STATUS_READY;         // OK prΩt
  3339.                         back_set_finished(sback, i);
  3340.                         back[i].r.notmodified=1;  // NON modifiΘ!
  3341.                         if ((opt->debug>0) && (opt->log!=NULL)) {
  3342.                           HTS_LOG(opt,LOG_DEBUG); fprintf(opt->log,"File loaded after test from cache: %s%s"LF,back[i].url_adr,back[i].url_fil); test_flush;
  3343.                         }
  3344.  
  3345.                         // finalize
  3346.                         //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
  3347.                         if (back[i].r.statuscode>0) {
  3348.                           back_finalize(opt,cache,sback,i);
  3349.                         }
  3350.                         
  3351. #if DEBUGCA
  3352.                         printf("..document α jour aprΦs requΦte: %s%s\n",back[i].url_adr,back[i].url_fil);
  3353. #endif
  3354.                         
  3355.                         //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
  3356.                       } else {  // erreur
  3357.                         back[i].status=STATUS_READY;  // terminΘ
  3358.                         back_set_finished(sback, i);
  3359.                         //printf("erreur cache\n");
  3360.                         
  3361.                       } 
  3362.                       
  3363. /********** NO - must complete the body! ********** */
  3364. #if 0
  3365.                     } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
  3366.                       || (back[i].r.statuscode==412)
  3367.                       || (back[i].r.statuscode==416)
  3368.                       ) {   // Ne pas prendre le html, erreurs connues et gΘrΘes
  3369. #if HTS_DEBUG_CLOSESOCK
  3370.                       DEBUG_W("back_wait(301,302,303,307,412,416..): deletehttp\n");
  3371. #endif
  3372.                       // Couper connexion
  3373.                       /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;*/
  3374.                       back_maydeletehttp(opt, cache, sback, i);
  3375.  
  3376.                       back[i].status=STATUS_READY;  // terminΘ
  3377.                       back_set_finished(sback, i);
  3378.                       // finalize
  3379.                       if (back[i].r.statuscode>0) {
  3380.                         back_finalize(opt,cache,sback,i);
  3381.                       }
  3382. #endif
  3383. /********** **************************** ********** */
  3384.                     } else {    // il faut aller le chercher
  3385.                       
  3386.                       // effacer buffer (requΦte)
  3387.                       if (!noFreebuff) {
  3388.                         deleteaddr(&back[i].r);
  3389.                         back[i].r.size=0;
  3390.                       }
  3391.                       
  3392.                       // traiter 206 (partial content)
  3393.                       // xxc SI CHUNK VERIFIER QUE CA MARCHE??
  3394.                       if (back[i].r.statuscode==206) {  // on nous envoie un morceau (la fin) coz une partie sur disque!
  3395.                         off_t sz=fsize(back[i].url_sav);
  3396. #if HDEBUG
  3397.                         printf("partial content: "LLintP" on disk..\n",(LLint)sz);
  3398. #endif
  3399.                         if (sz>=0) {
  3400.                           if (!is_hypertext_mime(opt,back[i].r.contenttype, back[i].url_sav)) {    // pas HTML
  3401.                             if (opt->getmode&2) {    // on peut ecrire des non html  **sinon ben euhh sera interceptΘ plus loin, donc rap sur ce qui va sortir**
  3402.                               filenote(&opt->state.strc,back[i].url_sav,NULL);    // noter fichier comme connu
  3403.                               file_notify(opt,back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 1, back[i].r.notmodified);
  3404.                               back[i].r.out=fopen(fconv(catbuff,back[i].url_sav),"ab");  // append
  3405.                               if (back[i].r.out) {
  3406.                                 back[i].r.is_write=1;    // Θcrire
  3407.                                 back[i].r.size=sz;    // dΘja Θcrit
  3408.                                 back[i].r.statuscode=HTTP_OK;  // Forcer 'OK'
  3409.                                 if (back[i].r.totalsize>0)
  3410.                                   back[i].r.totalsize+=sz;    // plus en fait
  3411.                                 fseek(back[i].r.out,0,SEEK_END);  // α la fin
  3412. #if HDEBUG
  3413.                                 printf("continue interrupted file\n");
  3414. #endif
  3415.                               } else {    // On est dans la m**
  3416.                                 back[i].status=STATUS_READY;  // terminΘ (voir plus loin)
  3417.                                 back_set_finished(sback, i);
  3418.                                 strcpybuff(back[i].r.msg,"Can not open partial file");
  3419.                               }
  3420.                             }
  3421.                           } else {    // mΘmoire
  3422.                             FILE* fp=fopen(fconv(catbuff,back[i].url_sav),"rb");
  3423.                             if (fp) {
  3424.                               LLint alloc_mem=sz + 1;
  3425.                               if (back[i].r.totalsize>0)
  3426.                                 alloc_mem+=back[i].r.totalsize;            // AJOUTER RESTANT!
  3427.                               if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct((size_t)alloc_mem)) ) {
  3428.                                 back[i].r.size=sz;
  3429.                                 if (back[i].r.totalsize>0)
  3430.                                   back[i].r.totalsize+=sz;    // plus en fait
  3431.                                 if (( fread(back[i].r.adr,1,sz,fp)) != sz) {
  3432.                                   back[i].status=STATUS_READY;  // terminΘ (voir plus loin)
  3433.                                   back_set_finished(sback, i);
  3434.                                   strcpybuff(back[i].r.msg,"Can not read partial file");
  3435.                                 } else {
  3436.                                   back[i].r.statuscode=HTTP_OK;  // Forcer 'OK'
  3437. #if HDEBUG
  3438.                                   printf("continue in mem interrupted file\n");
  3439. #endif
  3440.                                 }
  3441.                               } else {
  3442.                                 back[i].status=STATUS_READY;  // terminΘ (voir plus loin)
  3443.                                 back_set_finished(sback, i);
  3444.                                 strcpybuff(back[i].r.msg,"No memory for partial file");
  3445.                               }
  3446.                               fclose(fp);
  3447.                             } else {  // Argh.. 
  3448.                               back[i].status=STATUS_READY;  // terminΘ (voir plus loin)
  3449.                               back_set_finished(sback, i);
  3450.                               strcpybuff(back[i].r.msg,"Can not open partial file");
  3451.                             }
  3452.                           }
  3453.                         } else {    // Non trouvΘ??
  3454.                           back[i].status=STATUS_READY;  // terminΘ (voir plus loin)
  3455.                           back_set_finished(sback, i);
  3456.                           strcpybuff(back[i].r.msg,"Can not find partial file");
  3457.                         }
  3458.                         // Erreur?
  3459.                         if (back[i].status==STATUS_READY) {
  3460.                           if (back[i].r.soc!=INVALID_SOCKET) {
  3461. #if HTS_DEBUG_CLOSESOCK
  3462.                             DEBUG_W("back_wait(206 solve problems): deletehttp\n");
  3463. #endif
  3464.                             deletehttp(&back[i].r);
  3465.                           }
  3466.                           back[i].r.soc=INVALID_SOCKET;
  3467.                           //back[i].r.statuscode=206;  ????????
  3468.                           back[i].r.statuscode=STATUSCODE_NON_FATAL;
  3469.                           if (strnotempty(back[i].r.msg))
  3470.                             strcpybuff(back[i].r.msg,"Error attempting to solve status 206 (partial file)");
  3471.                         }
  3472.                       }
  3473.                       
  3474.                       if (back[i].status!=0) {  // non terminΘ (erreur)
  3475.                         if (!back[i].testmode) {    // fichier normal
  3476.                           
  3477.                           if (back[i].r.empty /* ?? && back[i].r.statuscode==HTTP_OK */) {  // empty response
  3478.                             // Couper connexion
  3479.                             back_maydeletehttp(opt, cache, sback, i);
  3480.                             /* KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
  3481.                             back[i].status=STATUS_READY;  // terminΘ
  3482.                             back_set_finished(sback, i);
  3483.                             if ( deleteaddr(&back[i].r) && (back[i].r.adr=(char*) malloct( 2)) ) {
  3484.                               back[i].r.adr[0] = 0;
  3485.                             }
  3486.                             back_finalize(opt,cache,sback,i);
  3487.                           }
  3488.                           else if (!back[i].r.is_chunk) {    // pas de chunk
  3489.                             //if (back[i].r.http11!=2) {    // pas de chunk
  3490.                             back[i].is_chunk=0;
  3491.                             back[i].status=1;     // start body
  3492.                           } else {
  3493. #if CHUNKDEBUG==1
  3494.                             printf("[%d] chunk encoding detected %s..\n",(int)back[i].r.soc, back[i].url_fil);
  3495. #endif
  3496.                             back[i].is_chunk=1;
  3497.                             back[i].chunk_adr=NULL;
  3498.                             back[i].chunk_size=0;
  3499.                             back[i].chunk_blocksize=0;
  3500.                             back[i].status=STATUS_CHUNK_WAIT;    // start body wait chunk
  3501.                             back[i].r.totalsize=0;   /* devalidate size! (rfc) */
  3502.                           }
  3503.                           if (back[i].rateout>0) {
  3504.                             back[i].rateout_time=time_local();  // refresh pour transfer rate
  3505.                           }
  3506. #if HDEBUG
  3507.                           printf("(buffer) start body!\n");
  3508. #endif
  3509.                         } else {     // mode test, ne pas passer en 1!!
  3510.                           back[i].status=STATUS_READY;    // READY
  3511.                           back_set_finished(sback, i);
  3512. #if HTS_DEBUG_CLOSESOCK
  3513.                           DEBUG_W("back_wait(test ok): deletehttp\n");
  3514. #endif
  3515.                           deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET;
  3516.                           if (back[i].r.statuscode==HTTP_OK) {
  3517.                             strcpybuff(back[i].r.msg,"Test: OK");
  3518.                             back[i].r.statuscode=STATUSCODE_TEST_OK;    // test rΘussi
  3519.                           }
  3520.                           else {    // test a ΘchouΘ, on ne change rien sauf que l'erreur est α titre indicatif
  3521.                             char tempo[1000];
  3522.                             strcpybuff(tempo,back[i].r.msg);
  3523.                             strcpybuff(back[i].r.msg,"Test: ");
  3524.                             strcatbuff(back[i].r.msg,tempo);
  3525.                           }
  3526.                           
  3527.                         }
  3528.                       }
  3529.                       
  3530.                       } 
  3531.                       
  3532.                       /*}*/
  3533.                       
  3534.                   }  // si LF
  3535.                 }  // r.size>2
  3536.               }  // si == 99
  3537.               
  3538.             } // si pas d'erreurs
  3539. #if BDEBUG==1
  3540.             printf("bytes overall: %d\n",back[i].r.size);
  3541. #endif
  3542.           }  // donnΘes dispo
  3543.           
  3544.           // en cas d'erreur cl, supprimer Θventuel fichier sur disque
  3545. #if HTS_REMOVE_BAD_FILES
  3546.           if (back[i].status<0) {
  3547.             if (!back[i].testmode) {    // pas en test
  3548.               unlink(back[i].url_sav);    // Θliminer fichier (endommagΘ)
  3549.               //printf("&& %s\n",back[i].url_sav);
  3550.             }
  3551.           }
  3552. #endif
  3553.  
  3554.           /* funny log for commandline users */
  3555.           //if (!opt->quiet) {  
  3556.           // petite animation
  3557.           if (opt->verbosedisplay==1) {
  3558.             if (back[i].status==STATUS_READY) {
  3559.               if (back[i].r.statuscode==HTTP_OK)
  3560.                 printf("* %s%s ("LLintP" bytes) - OK"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size);
  3561.               else
  3562.                 printf("* %s%s ("LLintP" bytes) - %d"VT_CLREOL"\r",back[i].url_adr,back[i].url_fil,(LLint)back[i].r.size,back[i].r.statuscode);
  3563.               fflush(stdout);
  3564.             }
  3565.           }
  3566.           //}
  3567.           
  3568.  
  3569.       } // status>0
  3570.     }  // for
  3571.     
  3572.     // vΘrifier timeouts
  3573.     if (gestion_timeout) {
  3574.       TStamp act;
  3575.       act=time_local();    // temps en secondes
  3576.       for(i_mod = 0 ; i_mod < (unsigned int) back_max ; i_mod++) {
  3577.       // for(i=0;i<back_max;i++) {
  3578.         unsigned int i = ( i_mod + mod_random ) % ( back_max );     
  3579.         if (back[i].status>0) {  // rΘception/connexion/..
  3580.           if (back[i].timeout>0) {
  3581.             //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
  3582.             if (((int) (act-back[i].timeout_refresh))>=back[i].timeout) {
  3583.               if (back[i].r.soc!=INVALID_SOCKET) {
  3584. #if HTS_DEBUG_CLOSESOCK
  3585.                 DEBUG_W("back_wait(timeout): deletehttp\n");
  3586. #endif
  3587.                 deletehttp(&back[i].r);
  3588.               }
  3589.               back[i].r.soc=INVALID_SOCKET;
  3590.               back[i].r.statuscode=STATUSCODE_TIMEOUT;
  3591.               if (back[i].status==STATUS_CONNECTING)
  3592.                 strcpybuff(back[i].r.msg,"Connect Time Out");
  3593.               else if (back[i].status==STATUS_WAIT_DNS)
  3594.                 strcpybuff(back[i].r.msg,"DNS Time Out");
  3595.               else
  3596.                 strcpybuff(back[i].r.msg,"Receive Time Out");
  3597.               back[i].status=STATUS_READY;  // terminΘ
  3598.               back_set_finished(sback, i);
  3599.             } else if ((back[i].rateout>0) && (back[i].status<99)) {
  3600.               if (((int) (act-back[i].rateout_time))>=HTS_WATCHRATE) {   // checker au bout de 15s
  3601.                 if ( (int) ((back[i].r.size)/(act-back[i].rateout_time)) < back[i].rateout ) {  // trop lent
  3602.                   back[i].status=STATUS_READY;  // terminΘ
  3603.                   back_set_finished(sback, i);
  3604.                   if (back[i].r.soc!=INVALID_SOCKET) {
  3605. #if HTS_DEBUG_CLOSESOCK
  3606.                     DEBUG_W("back_wait(rateout): deletehttp\n");
  3607. #endif
  3608.                     deletehttp(&back[i].r);
  3609.                   }
  3610.                   back[i].r.soc=INVALID_SOCKET;
  3611.                   back[i].r.statuscode=STATUSCODE_SLOW;
  3612.                   strcpybuff(back[i].r.msg,"Transfer Rate Too Low");
  3613.                 }
  3614.               }
  3615.             }
  3616.           }
  3617.         }
  3618.       }
  3619.     }
  3620.     max_loop--;
  3621.     max_loop_chk++;
  3622.   } while((busy_state) && (busy_recv) && (max_loop>0));
  3623.   if ((!busy_recv) && (!busy_state)) {
  3624.     if (max_loop_chk>=1) {
  3625.       Sleep(10);    // un tite pause pour Θviter les lag..
  3626.     }
  3627.   }
  3628. }
  3629.  
  3630. int back_checksize(httrackp* opt,lien_back* eback,int check_only_totalsize) {
  3631.   LLint size_to_test;
  3632.   if (check_only_totalsize)
  3633.     size_to_test=eback->r.totalsize;
  3634.   else
  3635.     size_to_test=max(eback->r.totalsize,eback->r.size);
  3636.   if (size_to_test>=0) {
  3637.     
  3638.     /* Interdiction taille par le wizard? */
  3639.     if (hts_testlinksize(opt,eback->url_adr,eback->url_fil,eback->r.totalsize/1024)==-1) {
  3640.       return 0;     /* interdit */
  3641.     }                     
  3642.     
  3643.     /* vΘrifier taille classique (heml et non html) */
  3644.     if ((istoobig(opt,size_to_test,eback->maxfile_html,eback->maxfile_nonhtml,eback->r.contenttype))) {
  3645.       return 0;     /* interdit */
  3646.     }
  3647.   }
  3648.   return 1;
  3649. }
  3650.  
  3651. int back_checkmirror(httrackp* opt) {
  3652.   // Check max time
  3653.   if ((opt->maxsite>0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
  3654.     if (opt->log) {
  3655.       fprintf(opt->log,"More than "LLintP" bytes have been transfered.. giving up"LF,(LLint)opt->maxsite);
  3656.       test_flush;
  3657.     } 
  3658.     return 0;
  3659.   } else if ((opt->maxtime>0) && ((time_local()-HTS_STAT.stat_timestart)>opt->maxtime)) {            
  3660.     if (opt->log) {
  3661.       fprintf(opt->log,"More than %d seconds passed.. giving up"LF,opt->maxtime);
  3662.       test_flush;
  3663.     } 
  3664.     return 0;
  3665.   }
  3666.   return 1;   /* Ok, go on */
  3667. }
  3668.  
  3669.  
  3670. // octets transfΘrΘs + add
  3671. LLint back_transfered(LLint nb,struct_back* sback) {
  3672.   lien_back* const back = sback->lnk;
  3673.   const int back_max = sback->count;
  3674.   int i;
  3675.   // ajouter octets en instance
  3676.   for(i=0;i<back_max;i++)
  3677.     if ((back[i].status>0) && (back[i].status<99 || back[i].status>=1000))
  3678.       nb += back[i].r.size;
  3679.   // stored (ready) slots
  3680.   if (sback->ready != NULL) {
  3681. #ifndef HTS_NO_BACK_ON_DISK
  3682.         nb += sback->ready_size_bytes;
  3683. #else
  3684.     struct_inthash_enum e = inthash_enum_new(sback->ready);
  3685.     inthash_chain* item;
  3686.     while((item = inthash_enum_next(&e))) {
  3687.       lien_back* ritem = (lien_back*) item->value.ptr;
  3688.       if ((ritem->status>0) && (ritem->status<99 || ritem->status>=1000))
  3689.         nb += ritem->r.size;
  3690.     }
  3691. #endif
  3692.   }
  3693.   return nb;      
  3694. }
  3695.  
  3696. // infos backing
  3697. // j: 1 afficher sockets 2 afficher autres 3 tout afficher
  3698. void back_info(struct_back* sback,int i,int j,FILE* fp) {
  3699.   lien_back* const back = sback->lnk;
  3700.   const int back_max = sback->count;
  3701.   assertf(i >= 0 && i < back_max);
  3702.   if (back[i].status>=0) {
  3703.     char BIGSTK s[HTS_URLMAXSIZE*2+1024]; 
  3704.     s[0]='\0';
  3705.     back_infostr(sback,i,j,s);
  3706.     strcatbuff(s,LF);
  3707.     fprintf(fp,"%s",s);
  3708.   }
  3709. }
  3710.  
  3711. // infos backing
  3712. // j: 1 afficher sockets 2 afficher autres 3 tout afficher
  3713. void back_infostr(struct_back* sback,int i,int j,char* s) {
  3714.   lien_back* const back = sback->lnk;
  3715.   const int back_max = sback->count;
  3716.   assertf(i >= 0 && i < back_max);
  3717.   if (back[i].status>=0) {
  3718.     int aff=0;
  3719.     if (j & 1) {
  3720.       if (back[i].status==STATUS_CONNECTING) {
  3721.         strcatbuff(s,"CONNECT ");
  3722.       } else if (back[i].status==STATUS_WAIT_HEADERS) {
  3723.         strcatbuff(s,"INFOS ");
  3724.         aff=1;
  3725.       } else if (back[i].status==STATUS_CHUNK_WAIT || back[i].status==STATUS_CHUNK_CR) {
  3726.         strcatbuff(s,"INFOSC");             // infos chunk
  3727.         aff=1;
  3728.       }
  3729.       else if (back[i].status>0) {
  3730.         strcatbuff(s,"RECEIVE "); 
  3731.         aff=1; 
  3732.       }
  3733.     } 
  3734.     if (j & 2) {
  3735.       if (back[i].status==STATUS_READY) {
  3736.         switch (back[i].r.statuscode) {
  3737.         case 200:
  3738.           strcatbuff(s,"READY ");
  3739.           aff=1;
  3740.           break;
  3741.         case -1:
  3742.           strcatbuff(s,"ERROR ");
  3743.           aff=1;
  3744.           break;
  3745.         case -2:
  3746.           strcatbuff(s,"TIMEOUT ");
  3747.           aff=1;
  3748.           break;
  3749.         case -3:
  3750.           strcatbuff(s,"TOOSLOW ");
  3751.           aff=1;
  3752.           break;
  3753.         case 400:
  3754.           strcatbuff(s,"BADREQUEST ");
  3755.           aff=1;
  3756.           break;
  3757.         case 401: case 403:
  3758.           strcatbuff(s,"FORBIDDEN ");
  3759.           aff=1;
  3760.           break;
  3761.         case 404:
  3762.           strcatbuff(s,"NOT FOUND ");
  3763.           aff=1;
  3764.           break;
  3765.         case 500:
  3766.           strcatbuff(s,"SERVERROR ");
  3767.           aff=1;
  3768.           break;
  3769.         default:
  3770.           {
  3771.             char s2[256];
  3772.             sprintf(s2,"ERROR(%d)",back[i].r.statuscode);
  3773.             strcatbuff(s,s2);
  3774.           }
  3775.           aff=1;
  3776.         }
  3777.       }
  3778.     }
  3779.     
  3780.     if (aff) {
  3781.       {
  3782.         char BIGSTK s2[HTS_URLMAXSIZE*2+1024];
  3783.         sprintf(s2,"\"%s",back[i].url_adr); strcatbuff(s,s2);
  3784.         
  3785.         if (back[i].url_fil[0]!='/') strcatbuff(s,"/");
  3786.         sprintf(s2,"%s\" ",back[i].url_fil); strcatbuff(s,s2);
  3787.         sprintf(s,LLintP" "LLintP" ",(LLint)back[i].r.size,(LLint)back[i].r.totalsize); strcatbuff(s,s2);
  3788.       }
  3789.     }
  3790.   }
  3791. }
  3792.  
  3793. // -- backing --
  3794.  
  3795. #undef test_flush
  3796.